From 0f248e0988b24bf4fcdaadd0e47127462206711e Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 3 Oct 2020 12:31:17 +0200 Subject: [PATCH 01/22] std: Make file copy ops use zero-copy mechanisms Use copy_file_range on Linux (if available), fcopyfile on Darwin, sendfile on *BSDs (and on Linux kernels without copy_file_range). --- lib/std/c/darwin.zig | 10 +++++ lib/std/fs.zig | 2 +- lib/std/os.zig | 101 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 102 insertions(+), 11 deletions(-) diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig index ed1ddb7d91..5b305d60e6 100644 --- a/lib/std/c/darwin.zig +++ b/lib/std/c/darwin.zig @@ -18,6 +18,16 @@ pub extern "c" fn _dyld_get_image_header(image_index: u32) ?*mach_header; pub extern "c" fn _dyld_get_image_vmaddr_slide(image_index: u32) usize; pub extern "c" fn _dyld_get_image_name(image_index: u32) [*:0]const u8; +pub const COPYFILE_ACL = 1 << 0; +pub const COPYFILE_STAT = 1 << 1; +pub const COPYFILE_XATTR = 1 << 2; +pub const COPYFILE_DATA = 1 << 3; + +pub const copyfile_state_t = *@Type(.Opaque); +pub extern "c" fn copyfile_state_alloc() copyfile_state_t; +pub extern "c" fn copyfile_state_free(state: copyfile_state_t) c_int; +pub extern "c" fn fcopyfile(from: fd_t, to: fd_t, state: ?copyfile_state_t, flags: u32) c_int; + pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8; pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, basep: *i64) isize; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 1890d7e136..75f0709df6 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1823,7 +1823,7 @@ pub const Dir = struct { var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode }); defer atomic_file.deinit(); - try atomic_file.file.writeFileAll(in_file, .{ .in_len = size }); + try os.copy_file(in_file.handle, atomic_file.file.handle, .{ .file_size = size }); return atomic_file.finish(); } diff --git a/lib/std/os.zig b/lib/std/os.zig index c06ce4ed00..c8d27eaa85 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4981,19 +4981,15 @@ pub const CopyFileRangeError = error{ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize { const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok; - // TODO support for other systems than linux - const try_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) != false; - - if (use_c or try_syscall) { + if (std.Target.current.os.tag == .linux and + (use_c or has_copy_file_range_syscall.get() != 0)) + { const sys = if (use_c) std.c else linux; var off_in_copy = @bitCast(i64, off_in); var off_out_copy = @bitCast(i64, off_out); const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags); - - // TODO avoid wasting a syscall every time if kernel is too old and returns ENOSYS https://github.com/ziglang/zig/issues/1018 - switch (sys.getErrno(rc)) { 0 => return @intCast(usize, rc), EBADF => unreachable, @@ -5005,9 +5001,14 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len EOVERFLOW => return error.Unseekable, EPERM => return error.PermissionDenied, ETXTBSY => return error.FileBusy, - EINVAL => {}, // these may not be regular files, try fallback - EXDEV => {}, // support for cross-filesystem copy added in Linux 5.3, use fallback - ENOSYS => {}, // syscall added in Linux 4.5, use fallback + // these may not be regular files, try fallback + EINVAL => {}, + // support for cross-filesystem copy added in Linux 5.3, use fallback + EXDEV => {}, + // syscall added in Linux 4.5, use fallback + ENOSYS => { + has_copy_file_range_syscall.set(0); + }, else => |err| return unexpectedErrno(err), } } @@ -5021,6 +5022,86 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len return pwrite(fd_out, buf[0..amt_read], off_out); } +var has_copy_file_range_syscall = std.atomic.Int(u1).init(1); + +pub const CopyFileOptions = struct { + /// Size in bytes of the source files, if available saves a call to stat(). + file_size: ?u64 = null, +}; + +pub const CopyFileError = error{ + SystemResources, + FileTooBig, + InputOutput, + IsDir, + OutOfMemory, + NoSpaceLeft, + Unseekable, + PermissionDenied, + FileBusy, +} || FStatError || SendFileError; + +/// Transfer all the data between two file descriptors in the most efficient way. +/// No metadata is transferred over. +pub fn copy_file(fd_in: fd_t, fd_out: fd_t, options: CopyFileOptions) CopyFileError!void { + if (comptime std.Target.current.isDarwin()) { + const rc = system.fcopyfile(fd_in, fd_out, null, system.COPYFILE_DATA); + switch (errno(rc)) { + 0 => return, + EINVAL => unreachable, + ENOMEM => return error.SystemResources, + // The source file was not a directory, symbolic link, or regular file. + // Try with the fallback path before giving up. + ENOTSUP => {}, + else => |err| return unexpectedErrno(err), + } + } + + const src_file_size = options.file_size orelse + @bitCast(u64, (try fstat(fd_in)).size); + var remaining = src_file_size; + + if (std.Target.current.os.tag == .linux) { + // Try copy_file_range first as that works at the FS level and is the + // most efficient method (if available). + if (has_copy_file_range_syscall.get() != 0) { + cfr_loop: while (remaining > 0) { + const copy_amt = math.cast(usize, remaining) catch math.maxInt(usize); + const rc = linux.copy_file_range(fd_in, null, fd_out, null, copy_amt, 0); + switch (errno(rc)) { + 0 => {}, + EBADF => unreachable, + EFBIG => return error.FileTooBig, + EIO => return error.InputOutput, + EISDIR => return error.IsDir, + ENOMEM => return error.OutOfMemory, + ENOSPC => return error.NoSpaceLeft, + EOVERFLOW => return error.Unseekable, + EPERM => return error.PermissionDenied, + ETXTBSY => return error.FileBusy, + // these may not be regular files, try fallback + EINVAL => break :cfr_loop, + // support for cross-filesystem copy added in Linux 5.3, use fallback + EXDEV => break :cfr_loop, + // syscall added in Linux 4.5, use fallback + ENOSYS => { + has_copy_file_range_syscall.set(0); + break :cfr_loop; + }, + else => |err| return unexpectedErrno(err), + } + remaining -= rc; + } + return; + } + } + + // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the + // fallback code will copy the contents chunk by chunk. + const empty_iovec = [0]iovec_const{}; + _ = try sendfile(fd_out, fd_in, 0, remaining, &empty_iovec, &empty_iovec, 0); +} + pub const PollError = error{ /// The kernel had no space to allocate file descriptor tables. SystemResources, From 8b4f5f039df65980d0a0ec6add1caf4c9bf2468c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 3 Oct 2020 19:51:22 +0200 Subject: [PATCH 02/22] Alternative strategy to avoid calling stat() This is an optimization as it avoids an extra syscall, but it's also a workaround for fstat being not available on Windows. --- lib/std/fs.zig | 2 +- lib/std/os.zig | 39 +++++++++++++++++++++------------------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 75f0709df6..c00976e01c 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1823,7 +1823,7 @@ pub const Dir = struct { var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode }); defer atomic_file.deinit(); - try os.copy_file(in_file.handle, atomic_file.file.handle, .{ .file_size = size }); + try os.copy_file(in_file.handle, atomic_file.file.handle, .{}); return atomic_file.finish(); } diff --git a/lib/std/os.zig b/lib/std/os.zig index c8d27eaa85..60845a3b87 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -5024,12 +5024,10 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len var has_copy_file_range_syscall = std.atomic.Int(u1).init(1); -pub const CopyFileOptions = struct { - /// Size in bytes of the source files, if available saves a call to stat(). - file_size: ?u64 = null, -}; +pub const CopyFileOptions = struct {}; pub const CopyFileError = error{ + BadFileHandle, SystemResources, FileTooBig, InputOutput, @@ -5057,20 +5055,18 @@ pub fn copy_file(fd_in: fd_t, fd_out: fd_t, options: CopyFileOptions) CopyFileEr } } - const src_file_size = options.file_size orelse - @bitCast(u64, (try fstat(fd_in)).size); - var remaining = src_file_size; - if (std.Target.current.os.tag == .linux) { // Try copy_file_range first as that works at the FS level and is the // most efficient method (if available). if (has_copy_file_range_syscall.get() != 0) { - cfr_loop: while (remaining > 0) { - const copy_amt = math.cast(usize, remaining) catch math.maxInt(usize); - const rc = linux.copy_file_range(fd_in, null, fd_out, null, copy_amt, 0); + cfr_loop: while (true) { + // The kernel checks `file_pos+count` for overflow, use a 32 bit + // value so that the syscall won't return EINVAL except for + // impossibly large files. + const rc = linux.copy_file_range(fd_in, null, fd_out, null, math.maxInt(u32), 0); switch (errno(rc)) { 0 => {}, - EBADF => unreachable, + EBADF => return error.BadFileHandle, EFBIG => return error.FileTooBig, EIO => return error.InputOutput, EISDIR => return error.IsDir, @@ -5079,27 +5075,34 @@ pub fn copy_file(fd_in: fd_t, fd_out: fd_t, options: CopyFileOptions) CopyFileEr EOVERFLOW => return error.Unseekable, EPERM => return error.PermissionDenied, ETXTBSY => return error.FileBusy, - // these may not be regular files, try fallback + // These may not be regular files, try fallback EINVAL => break :cfr_loop, - // support for cross-filesystem copy added in Linux 5.3, use fallback + // Support for cross-filesystem copy added in Linux 5.3, use fallback EXDEV => break :cfr_loop, - // syscall added in Linux 4.5, use fallback + // Syscall added in Linux 4.5, use fallback ENOSYS => { has_copy_file_range_syscall.set(0); break :cfr_loop; }, else => |err| return unexpectedErrno(err), } - remaining -= rc; + // Terminate when no data was copied + if (rc == 0) return; } - return; + // This point is reached when an error occurred, hopefully no data + // was transferred yet } } // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the // fallback code will copy the contents chunk by chunk. const empty_iovec = [0]iovec_const{}; - _ = try sendfile(fd_out, fd_in, 0, remaining, &empty_iovec, &empty_iovec, 0); + var offset: u64 = 0; + sendfile_loop: while (true) { + const amt = try sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0); + if (amt == 0) break :sendfile_loop; + offset += amt; + } } pub const PollError = error{ From a419a1aabce63fedcc77a1118d596864fcff46e3 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 6 Oct 2020 09:38:59 +0200 Subject: [PATCH 03/22] Move copy_file to fs namespace Now it is a private API. Also handle short writes in copy_file_range fallback implementation. --- lib/std/fs.zig | 48 ++++++++++++++++++++- lib/std/os.zig | 110 ++++++++++--------------------------------------- 2 files changed, 69 insertions(+), 89 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index c00976e01c..92f68590f9 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1823,7 +1823,7 @@ pub const Dir = struct { var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode }); defer atomic_file.deinit(); - try os.copy_file(in_file.handle, atomic_file.file.handle, .{}); + try copy_file(in_file.handle, atomic_file.file.handle); return atomic_file.finish(); } @@ -2263,6 +2263,52 @@ pub fn realpathAlloc(allocator: *Allocator, pathname: []const u8) ![]u8 { return allocator.dupe(u8, try os.realpath(pathname, &buf)); } +const CopyFileError = error{SystemResources} || os.CopyFileRangeError || os.SendFileError; + +/// Transfer all the data between two file descriptors in the most efficient way. +/// No metadata is transferred over. +fn copy_file(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void { + if (comptime std.Target.current.isDarwin()) { + const rc = os.system.fcopyfile(fd_in, fd_out, null, os.system.COPYFILE_DATA); + switch (errno(rc)) { + 0 => return, + EINVAL => unreachable, + ENOMEM => return error.SystemResources, + // The source file was not a directory, symbolic link, or regular file. + // Try with the fallback path before giving up. + ENOTSUP => {}, + else => |err| return unexpectedErrno(err), + } + } + + if (std.Target.current.os.tag == .linux) { + // Try copy_file_range first as that works at the FS level and is the + // most efficient method (if available). + var offset: u64 = 0; + cfr_loop: while (true) { + // The kernel checks `offset+count` for overflow, use a 32 bit + // value so that the syscall won't return EINVAL except for + // impossibly large files. + const amt = try os.copy_file_range(fd_in, offset, fd_out, offset, math.maxInt(u32), 0); + // Terminate when no data was copied + if (amt == 0) break :cfr_loop; + offset += amt; + } + return; + } + + // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the + // fallback code will copy the contents chunk by chunk. + const empty_iovec = [0]os.iovec_const{}; + var offset: u64 = 0; + sendfile_loop: while (true) { + const amt = try os.sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0); + // Terminate when no data was copied + if (amt == 0) break :sendfile_loop; + offset += amt; + } +} + test "" { if (builtin.os.tag != .wasi) { _ = makeDirAbsolute; diff --git a/lib/std/os.zig b/lib/std/os.zig index 60845a3b87..e4e0b1d4a1 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4945,6 +4945,7 @@ pub fn sendfile( pub const CopyFileRangeError = error{ FileTooBig, InputOutput, + InvalidFileDescriptor, IsDir, OutOfMemory, NoSpaceLeft, @@ -4978,6 +4979,11 @@ pub const CopyFileRangeError = error{ /// Other systems fall back to calling `pread` / `pwrite`. /// /// Maximum offsets on Linux are `math.maxInt(i64)`. +var has_copy_file_range_syscall = init: { + const kernel_has_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true; + break :init std.atomic.Int(u1).init(@boolToInt(kernel_has_syscall)); +}; + pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize { const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok; @@ -4992,7 +4998,7 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags); switch (sys.getErrno(rc)) { 0 => return @intCast(usize, rc), - EBADF => unreachable, + EBADF => return error.InvalidFileDescriptor, EFBIG => return error.FileTooBig, EIO => return error.InputOutput, EISDIR => return error.IsDir, @@ -5013,96 +5019,24 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len } } - var buf: [8 * 4096]u8 = undefined; - const adjusted_count = math.min(buf.len, len); - const amt_read = try pread(fd_in, buf[0..adjusted_count], off_in); - // TODO without @as the line below fails to compile for wasm32-wasi: - // error: integer value 0 cannot be coerced to type 'os.PWriteError!usize' - if (amt_read == 0) return @as(usize, 0); - return pwrite(fd_out, buf[0..amt_read], off_out); -} + var buf: [2 * 4096]u8 = undefined; -var has_copy_file_range_syscall = std.atomic.Int(u1).init(1); - -pub const CopyFileOptions = struct {}; - -pub const CopyFileError = error{ - BadFileHandle, - SystemResources, - FileTooBig, - InputOutput, - IsDir, - OutOfMemory, - NoSpaceLeft, - Unseekable, - PermissionDenied, - FileBusy, -} || FStatError || SendFileError; - -/// Transfer all the data between two file descriptors in the most efficient way. -/// No metadata is transferred over. -pub fn copy_file(fd_in: fd_t, fd_out: fd_t, options: CopyFileOptions) CopyFileError!void { - if (comptime std.Target.current.isDarwin()) { - const rc = system.fcopyfile(fd_in, fd_out, null, system.COPYFILE_DATA); - switch (errno(rc)) { - 0 => return, - EINVAL => unreachable, - ENOMEM => return error.SystemResources, - // The source file was not a directory, symbolic link, or regular file. - // Try with the fallback path before giving up. - ENOTSUP => {}, - else => |err| return unexpectedErrno(err), - } + var total_copied: usize = 0; + var read_off = off_in; + var write_off = off_out; + while (total_copied < len) { + const adjusted_count = math.min(buf.len, len - total_copied); + const amt_read = try pread(fd_in, buf[0..adjusted_count], read_off); + if (amt_read == 0) break; + const amt_written = try pwrite(fd_out, buf[0..amt_read], write_off); + // pwrite may write less than the specified amount, handle the remaining + // chunk of data in the next iteration + read_off += amt_written; + write_off += amt_written; + total_copied += amt_written; } - if (std.Target.current.os.tag == .linux) { - // Try copy_file_range first as that works at the FS level and is the - // most efficient method (if available). - if (has_copy_file_range_syscall.get() != 0) { - cfr_loop: while (true) { - // The kernel checks `file_pos+count` for overflow, use a 32 bit - // value so that the syscall won't return EINVAL except for - // impossibly large files. - const rc = linux.copy_file_range(fd_in, null, fd_out, null, math.maxInt(u32), 0); - switch (errno(rc)) { - 0 => {}, - EBADF => return error.BadFileHandle, - EFBIG => return error.FileTooBig, - EIO => return error.InputOutput, - EISDIR => return error.IsDir, - ENOMEM => return error.OutOfMemory, - ENOSPC => return error.NoSpaceLeft, - EOVERFLOW => return error.Unseekable, - EPERM => return error.PermissionDenied, - ETXTBSY => return error.FileBusy, - // These may not be regular files, try fallback - EINVAL => break :cfr_loop, - // Support for cross-filesystem copy added in Linux 5.3, use fallback - EXDEV => break :cfr_loop, - // Syscall added in Linux 4.5, use fallback - ENOSYS => { - has_copy_file_range_syscall.set(0); - break :cfr_loop; - }, - else => |err| return unexpectedErrno(err), - } - // Terminate when no data was copied - if (rc == 0) return; - } - // This point is reached when an error occurred, hopefully no data - // was transferred yet - } - } - - // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the - // fallback code will copy the contents chunk by chunk. - const empty_iovec = [0]iovec_const{}; - var offset: u64 = 0; - sendfile_loop: while (true) { - const amt = try sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0); - if (amt == 0) break :sendfile_loop; - offset += amt; - } + return total_copied; } pub const PollError = error{ From 1f7ec0de706eded887bc8dbcf5f45a5546d1be5c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 6 Oct 2020 11:57:23 +0200 Subject: [PATCH 04/22] Address review comments & fix compilation errors --- lib/std/fs.zig | 18 +++++++++--------- lib/std/os.zig | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 92f68590f9..6b4f9384dd 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -2270,14 +2270,14 @@ const CopyFileError = error{SystemResources} || os.CopyFileRangeError || os.Send fn copy_file(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void { if (comptime std.Target.current.isDarwin()) { const rc = os.system.fcopyfile(fd_in, fd_out, null, os.system.COPYFILE_DATA); - switch (errno(rc)) { + switch (os.errno(rc)) { 0 => return, - EINVAL => unreachable, - ENOMEM => return error.SystemResources, - // The source file was not a directory, symbolic link, or regular file. + os.EINVAL => unreachable, + os.ENOMEM => return error.SystemResources, + // The source file is not a directory, symbolic link, or regular file. // Try with the fallback path before giving up. - ENOTSUP => {}, - else => |err| return unexpectedErrno(err), + os.ENOTSUP => {}, + else => |err| return os.unexpectedErrno(err), } } @@ -2286,9 +2286,9 @@ fn copy_file(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void { // most efficient method (if available). var offset: u64 = 0; cfr_loop: while (true) { - // The kernel checks `offset+count` for overflow, use a 32 bit - // value so that the syscall won't return EINVAL except for - // impossibly large files. + // The kernel checks the u64 value `offset+count` for overflow, use + // a 32 bit value so that the syscall won't return EINVAL except for + // impossibly large files (> 2^64-1 - 2^32-1). const amt = try os.copy_file_range(fd_in, offset, fd_out, offset, math.maxInt(u32), 0); // Terminate when no data was copied if (amt == 0) break :cfr_loop; diff --git a/lib/std/os.zig b/lib/std/os.zig index e4e0b1d4a1..0e81b73f8c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4954,6 +4954,11 @@ pub const CopyFileRangeError = error{ FileBusy, } || PReadError || PWriteError || UnexpectedError; +var has_copy_file_range_syscall = init: { + const kernel_has_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true; + break :init std.atomic.Int(bool).init(kernel_has_syscall); +}; + /// Transfer data between file descriptors at specified offsets. /// Returns the number of bytes written, which can less than requested. /// @@ -4979,16 +4984,11 @@ pub const CopyFileRangeError = error{ /// Other systems fall back to calling `pread` / `pwrite`. /// /// Maximum offsets on Linux are `math.maxInt(i64)`. -var has_copy_file_range_syscall = init: { - const kernel_has_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true; - break :init std.atomic.Int(u1).init(@boolToInt(kernel_has_syscall)); -}; - pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize { const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok; if (std.Target.current.os.tag == .linux and - (use_c or has_copy_file_range_syscall.get() != 0)) + (use_c or has_copy_file_range_syscall.get())) { const sys = if (use_c) std.c else linux; @@ -5013,7 +5013,7 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len EXDEV => {}, // syscall added in Linux 4.5, use fallback ENOSYS => { - has_copy_file_range_syscall.set(0); + has_copy_file_range_syscall.set(false); }, else => |err| return unexpectedErrno(err), } From 03762da2af8753ecf7f4bc2005dd00d570c51ae7 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Wed, 7 Oct 2020 11:13:26 +0200 Subject: [PATCH 05/22] New review round --- lib/std/c/darwin.zig | 2 -- lib/std/fs.zig | 5 +++-- lib/std/os.zig | 33 ++++++++++++--------------------- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig index 5b305d60e6..93efdd061c 100644 --- a/lib/std/c/darwin.zig +++ b/lib/std/c/darwin.zig @@ -24,8 +24,6 @@ pub const COPYFILE_XATTR = 1 << 2; pub const COPYFILE_DATA = 1 << 3; pub const copyfile_state_t = *@Type(.Opaque); -pub extern "c" fn copyfile_state_alloc() copyfile_state_t; -pub extern "c" fn copyfile_state_free(state: copyfile_state_t) c_int; pub extern "c" fn fcopyfile(from: fd_t, to: fd_t, state: ?copyfile_state_t, flags: u32) c_int; pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 6b4f9384dd..5f49bb3766 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -2265,8 +2265,9 @@ pub fn realpathAlloc(allocator: *Allocator, pathname: []const u8) ![]u8 { const CopyFileError = error{SystemResources} || os.CopyFileRangeError || os.SendFileError; -/// Transfer all the data between two file descriptors in the most efficient way. -/// No metadata is transferred over. +// Transfer all the data between two file descriptors in the most efficient way. +// The copy starts at offset 0, the initial offsets are preserved. +// No metadata is transferred over. fn copy_file(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void { if (comptime std.Target.current.isDarwin()) { const rc = os.system.fcopyfile(fd_in, fd_out, null, os.system.COPYFILE_DATA); diff --git a/lib/std/os.zig b/lib/std/os.zig index 0e81b73f8c..6e25451879 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -4945,7 +4945,9 @@ pub fn sendfile( pub const CopyFileRangeError = error{ FileTooBig, InputOutput, - InvalidFileDescriptor, + /// `fd_in` is not open for reading; or `fd_out` is not open for writing; + /// or the `O_APPEND` flag is set for `fd_out`. + FilesOpenedWithWrongFlags, IsDir, OutOfMemory, NoSpaceLeft, @@ -4955,7 +4957,7 @@ pub const CopyFileRangeError = error{ } || PReadError || PWriteError || UnexpectedError; var has_copy_file_range_syscall = init: { - const kernel_has_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true; + const kernel_has_syscall = std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true; break :init std.atomic.Int(bool).init(kernel_has_syscall); }; @@ -4998,7 +5000,7 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags); switch (sys.getErrno(rc)) { 0 => return @intCast(usize, rc), - EBADF => return error.InvalidFileDescriptor, + EBADF => return error.FilesOpenedWithWrongFlags, EFBIG => return error.FileTooBig, EIO => return error.InputOutput, EISDIR => return error.IsDir, @@ -5019,24 +5021,13 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len } } - var buf: [2 * 4096]u8 = undefined; - - var total_copied: usize = 0; - var read_off = off_in; - var write_off = off_out; - while (total_copied < len) { - const adjusted_count = math.min(buf.len, len - total_copied); - const amt_read = try pread(fd_in, buf[0..adjusted_count], read_off); - if (amt_read == 0) break; - const amt_written = try pwrite(fd_out, buf[0..amt_read], write_off); - // pwrite may write less than the specified amount, handle the remaining - // chunk of data in the next iteration - read_off += amt_written; - write_off += amt_written; - total_copied += amt_written; - } - - return total_copied; + var buf: [8 * 4096]u8 = undefined; + const adjusted_count = math.min(buf.len, len); + const amt_read = try pread(fd_in, buf[0..adjusted_count], off_in); + // TODO without @as the line below fails to compile for wasm32-wasi: + // error: integer value 0 cannot be coerced to type 'os.PWriteError!usize' + if (amt_read == 0) return @as(usize, 0); + return pwrite(fd_out, buf[0..amt_read], off_out); } pub const PollError = error{ From f0a73df8e72e156bd95fa6c7f4de9512513d01b3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 5 Oct 2020 21:59:07 +0200 Subject: [PATCH 06/22] Add prototype for export trie generation in MachO linker Signed-off-by: Jakub Konka --- src/link/MachO.zig | 136 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index a1b9484e13..5db71ee1ab 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -64,6 +64,97 @@ const LoadCommand = union(enum) { } }; +/// Represents export trie used in MachO executables and dynamic libraries. +/// The purpose of an export trie is to encode as compactly as possible all +/// export symbols for the loader `dyld`. +/// The export trie encodes offset and other information using ULEB128 +/// encoding, and is part of the __LINKEDIT segment. +const Trie = struct { + const Node = struct { + const Edge = struct { + from: *Node, + to: *Node, + label: []const u8, + + pub fn deinit(self: *Edge, alloc: *Allocator) void { + self.to.deinit(alloc); + alloc.destroy(self.to); + self.from = undefined; + self.to = undefined; + } + }; + + edges: std.ArrayListUnmanaged(Edge) = .{}, + + pub fn deinit(self: *Node, alloc: *Allocator) void { + for (self.edges.items) |*edge| { + edge.deinit(alloc); + } + self.edges.deinit(alloc); + } + + pub fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !void { + // Traverse all edges. + for (self.edges.items) |*edge| { + const match = mem.indexOfDiff(u8, edge.label, label) orelse return; // Got a full match, don't do anything. + if (match - prefix > 0) { + // If we match, we advance further down the trie. + return edge.to.put(alloc, edge, match, label); + } + } + + if (fromEdge) |from| { + if (mem.eql(u8, from.label, label[0..prefix])) { + if (prefix == label.len) return; + } else { + // Fixup nodes. We need to insert an intermediate node between + // from.to and self. + const mid = try alloc.create(Node); + mid.* = .{}; + const to_label = from.label; + from.to = mid; + from.label = label[0..prefix]; + + try mid.edges.append(alloc, .{ + .from = mid, + .to = self, + .label = to_label, + }); + + if (prefix == label.len) return; // We're done. + + const new_node = try alloc.create(Node); + new_node.* = .{}; + return mid.edges.append(alloc, .{ + .from = mid, + .to = new_node, + .label = label, + }); + } + } + + // Add a new edge. + const node = try alloc.create(Node); + node.* = .{}; + return self.edges.append(alloc, .{ + .from = self, + .to = node, + .label = label, + }); + } + }; + + root: Node, + + pub fn put(self: *Trie, alloc: *Allocator, word: []const u8) !void { + return self.root.put(alloc, null, 0, word); + } + + pub fn deinit(self: *Trie, alloc: *Allocator) void { + self.root.deinit(alloc); + } +}; + base: File, /// Table of all load commands @@ -1533,3 +1624,48 @@ fn satMul(a: anytype, b: anytype) @TypeOf(a, b) { const T = @TypeOf(a, b); return std.math.mul(T, a, b) catch std.math.maxInt(T); } + +test "Trie basic" { + const testing = @import("std").testing; + var gpa = testing.allocator; + + var trie: Trie = .{ + .root = .{}, + }; + defer trie.deinit(gpa); + + // root + testing.expect(trie.root.edges.items.len == 0); + + // root --- _st ---> node + try trie.put(gpa, "_st"); + testing.expect(trie.root.edges.items.len == 1); + testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st")); + + { + // root --- _st ---> node --- _start ---> node + try trie.put(gpa, "_start"); + testing.expect(trie.root.edges.items.len == 1); + + const nextEdge = &trie.root.edges.items[0]; + testing.expect(mem.eql(u8, nextEdge.label, "_st")); + testing.expect(nextEdge.to.edges.items.len == 1); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start")); + } + { + // root --- _ ---> node --- _st ---> node --- _start ---> node + // | + // | --- _main ---> node + try trie.put(gpa, "_main"); + testing.expect(trie.root.edges.items.len == 1); + + const nextEdge = &trie.root.edges.items[0]; + testing.expect(mem.eql(u8, nextEdge.label, "_")); + testing.expect(nextEdge.to.edges.items.len == 2); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st")); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main")); + + const nextNextEdge = &nextEdge.to.edges.items[0]; + testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start")); + } +} From e76fb8d8c82ffc9fdeef2de0a6008c756103811b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 6 Oct 2020 22:34:39 +0200 Subject: [PATCH 07/22] Add incomplete writing of trie to bytes buffer Signed-off-by: Jakub Konka --- src/link/MachO.zig | 101 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 5db71ee1ab..b522655b13 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -84,6 +84,8 @@ const Trie = struct { } }; + export_flags: ?u64 = null, + offset: ?u64 = null, edges: std.ArrayListUnmanaged(Edge) = .{}, pub fn deinit(self: *Node, alloc: *Allocator) void { @@ -93,10 +95,10 @@ const Trie = struct { self.edges.deinit(alloc); } - pub fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !void { + pub fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node { // Traverse all edges. for (self.edges.items) |*edge| { - const match = mem.indexOfDiff(u8, edge.label, label) orelse return; // Got a full match, don't do anything. + const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything. if (match - prefix > 0) { // If we match, we advance further down the trie. return edge.to.put(alloc, edge, match, label); @@ -105,7 +107,7 @@ const Trie = struct { if (fromEdge) |from| { if (mem.eql(u8, from.label, label[0..prefix])) { - if (prefix == label.len) return; + if (prefix == label.len) return self; } else { // Fixup nodes. We need to insert an intermediate node between // from.to and self. @@ -121,35 +123,86 @@ const Trie = struct { .label = to_label, }); - if (prefix == label.len) return; // We're done. + if (prefix == label.len) return self; // We're done. const new_node = try alloc.create(Node); new_node.* = .{}; - return mid.edges.append(alloc, .{ + + try mid.edges.append(alloc, .{ .from = mid, .to = new_node, .label = label, }); + + return new_node; } } // Add a new edge. const node = try alloc.create(Node); node.* = .{}; - return self.edges.append(alloc, .{ + + try self.edges.append(alloc, .{ .from = self, .to = node, .label = label, }); + + return node; + } + + pub fn write(self: Node, buf: []u8, offset: u64) error{NoSpaceLeft}!usize { + var pos: usize = 0; + if (self.offset) |off| { + var info_buf_pos: usize = 0; + var info_buf: [@sizeOf(u64) * 2]u8 = undefined; + info_buf_pos += try std.debug.leb.writeULEB128Mem(info_buf[0..], self.export_flags.?); + info_buf_pos += try std.debug.leb.writeULEB128Mem(info_buf[info_buf_pos..], off); + log.debug("info_buf = {x}\n", .{info_buf[0..info_buf_pos]}); + pos += try std.debug.leb.writeULEB128Mem(buf[pos..], info_buf_pos); + mem.copy(u8, buf[pos..], info_buf[0..info_buf_pos]); + pos += info_buf_pos; + log.debug("buf = {x}\n", .{buf}); + } else { + buf[pos] = 0; + pos += 1; + } + buf[pos] = @intCast(u8, self.edges.items.len); + pos += 1; + + for (self.edges.items) |edge| { + mem.copy(u8, buf[pos..], edge.label); + pos += edge.label.len; + buf[pos] = 0; + pos += 1; + const curr_offset = pos + offset + 1; + pos += try std.debug.leb.writeULEB128Mem(buf[pos..], curr_offset); + pos += try edge.to.write(buf[pos..], curr_offset); + log.debug("buf = {x}\n", .{buf}); + } + + return pos; } }; root: Node, - pub fn put(self: *Trie, alloc: *Allocator, word: []const u8) !void { + pub fn put(self: *Trie, alloc: *Allocator, word: []const u8) !*Node { return self.root.put(alloc, null, 0, word); } + pub fn write(self: Trie, alloc: *Allocator, file: *fs.File, offset: u64) !void { + // TODO get the actual node count + const count = 10; + const node_size = @sizeOf(u64) * 2; + + var buf = try alloc.alloc(u8, count * node_size); + defer alloc.free(buf); + + const written = try self.root.write(buf, 0); + return file.pwriteAll(buf[0..written], offset); + } + pub fn deinit(self: *Trie, alloc: *Allocator) void { self.root.deinit(alloc); } @@ -347,10 +400,10 @@ pub fn flushModule(self: *MachO, comp: *Compilation) !void { switch (self.base.options.output_mode) { .Exe => { - if (self.entry_addr) |addr| { - // Write export trie. - try self.writeExportTrie(); + // Write export trie. + try self.writeExportTrie(); + if (self.entry_addr) |addr| { // Update LC_MAIN with entry offset const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment; const main_cmd = &self.load_commands.items[self.main_cmd_index.?].EntryPoint; @@ -1474,25 +1527,25 @@ fn writeAllUndefSymbols(self: *MachO) !void { } fn writeExportTrie(self: *MachO) !void { - assert(self.entry_addr != null); + if (self.global_symbols.items.len == 0) return; // No exports, nothing to do. - // TODO implement mechanism for generating a prefix tree of the exported symbols - // single branch export trie - var buf = [_]u8{0} ** 24; - buf[0] = 0; // root node - buf[1] = 1; // 1 branch from root - mem.copy(u8, buf[2..], "_start"); - buf[8] = 0; - buf[9] = 9 + 1; + var trie: Trie = .{ + .root = .{}, + }; + defer trie.deinit(self.base.allocator); const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment; - const addr = self.entry_addr.? - text_segment.vmaddr; - const written = try std.debug.leb.writeULEB128Mem(buf[12..], addr); - buf[10] = @intCast(u8, written) + 1; - buf[11] = 0; + + for (self.global_symbols.items) |symbol| { + // TODO figure out if we should put all global symbols into the export trie + const name = self.getString(symbol.n_strx); + const node = try trie.put(self.base.allocator, name); + node.offset = symbol.n_value - text_segment.vmaddr; + node.export_flags = 0; // TODO workout creation of export flags + } const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo; - try self.base.file.?.pwriteAll(buf[0..], dyld_info.export_off); + try trie.write(self.base.allocator, &self.base.file.?, dyld_info.export_off); } fn writeStringTable(self: *MachO) !void { From b13b36a71d63b6dfe4beda940fa6f9488fb3690a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 7 Oct 2020 00:36:13 +0200 Subject: [PATCH 08/22] Approach using array list for auto mem mgmt Signed-off-by: Jakub Konka --- src/link/MachO.zig | 79 ++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index b522655b13..b49d022637 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -151,37 +151,49 @@ const Trie = struct { return node; } - pub fn write(self: Node, buf: []u8, offset: u64) error{NoSpaceLeft}!usize { - var pos: usize = 0; + pub fn write(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void { if (self.offset) |off| { - var info_buf_pos: usize = 0; + // Terminal node info: encode export flags and vmaddr offset of this symbol. + var info_buf_len: usize = 0; var info_buf: [@sizeOf(u64) * 2]u8 = undefined; - info_buf_pos += try std.debug.leb.writeULEB128Mem(info_buf[0..], self.export_flags.?); - info_buf_pos += try std.debug.leb.writeULEB128Mem(info_buf[info_buf_pos..], off); - log.debug("info_buf = {x}\n", .{info_buf[0..info_buf_pos]}); - pos += try std.debug.leb.writeULEB128Mem(buf[pos..], info_buf_pos); - mem.copy(u8, buf[pos..], info_buf[0..info_buf_pos]); - pos += info_buf_pos; - log.debug("buf = {x}\n", .{buf}); + info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[0..], self.export_flags.?); + info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[info_buf_len..], off); + + // Encode the size of the terminal node info. + var size_buf: [@sizeOf(u64)]u8 = undefined; + const size_buf_len = try std.debug.leb.writeULEB128Mem(size_buf[0..], info_buf_len); + + // Now, write them to the output buffer. + try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len); + buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]); + buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]); } else { - buf[pos] = 0; - pos += 1; + // Non-terminal node is delimited by 0 byte. + try buffer.append(alloc, 0); } - buf[pos] = @intCast(u8, self.edges.items.len); - pos += 1; + // Write number of edges (max legal number of edges is 256). + try buffer.append(alloc, @intCast(u8, self.edges.items.len)); - for (self.edges.items) |edge| { - mem.copy(u8, buf[pos..], edge.label); - pos += edge.label.len; - buf[pos] = 0; - pos += 1; - const curr_offset = pos + offset + 1; - pos += try std.debug.leb.writeULEB128Mem(buf[pos..], curr_offset); - pos += try edge.to.write(buf[pos..], curr_offset); - log.debug("buf = {x}\n", .{buf}); + var node_offset_info: [@sizeOf(u8)]u64 = undefined; + for (self.edges.items) |edge, i| { + // Write edges labels leaving out space in-between to later populate + // with offsets to each node. + try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte + buffer.appendSliceAssumeCapacity(edge.label); + buffer.appendAssumeCapacity(0); + node_offset_info[i] = buffer.items.len; + const padding = [_]u8{0} ** @sizeOf(u64); + buffer.appendSliceAssumeCapacity(padding[0..]); } - return pos; + for (self.edges.items) |edge, i| { + const offset = buffer.items.len; + try edge.to.write(alloc, buffer); + // We can now populate the offset to the node pointed by this edge. + var offset_buf: [@sizeOf(u64)]u8 = undefined; + const offset_buf_len = try std.debug.leb.writeULEB128Mem(offset_buf[0..], offset); + mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]); + } } }; @@ -191,16 +203,10 @@ const Trie = struct { return self.root.put(alloc, null, 0, word); } - pub fn write(self: Trie, alloc: *Allocator, file: *fs.File, offset: u64) !void { - // TODO get the actual node count - const count = 10; - const node_size = @sizeOf(u64) * 2; + pub const WriteError = error{ OutOfMemory, NoSpaceLeft }; - var buf = try alloc.alloc(u8, count * node_size); - defer alloc.free(buf); - - const written = try self.root.write(buf, 0); - return file.pwriteAll(buf[0..written], offset); + pub fn write(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void { + return self.root.write(alloc, buffer); } pub fn deinit(self: *Trie, alloc: *Allocator) void { @@ -1544,8 +1550,13 @@ fn writeExportTrie(self: *MachO) !void { node.export_flags = 0; // TODO workout creation of export flags } + var buffer: std.ArrayListUnmanaged(u8) = .{}; + defer buffer.deinit(self.base.allocator); + + try trie.write(self.base.allocator, &buffer); + const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo; - try trie.write(self.base.allocator, &self.base.file.?, dyld_info.export_off); + try self.base.file.?.pwriteAll(buffer.items, dyld_info.export_off); } fn writeStringTable(self: *MachO) !void { From bdab4f53c1fa614fcd89468f305184fa36520039 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 7 Oct 2020 19:36:50 +0200 Subject: [PATCH 09/22] Move trie structure into its own file-module Signed-off-by: Jakub Konka --- src/link/MachO.zig | 209 ++------------------------------ src/link/MachO/Trie.zig | 259 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 200 deletions(-) create mode 100644 src/link/MachO/Trie.zig diff --git a/src/link/MachO.zig b/src/link/MachO.zig index b49d022637..afc54f8f7b 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -20,6 +20,8 @@ const File = link.File; const Cache = @import("../Cache.zig"); const target_util = @import("../target.zig"); +const Trie = @import("MachO/Trie.zig"); + pub const base_tag: File.Tag = File.Tag.macho; const LoadCommand = union(enum) { @@ -64,156 +66,6 @@ const LoadCommand = union(enum) { } }; -/// Represents export trie used in MachO executables and dynamic libraries. -/// The purpose of an export trie is to encode as compactly as possible all -/// export symbols for the loader `dyld`. -/// The export trie encodes offset and other information using ULEB128 -/// encoding, and is part of the __LINKEDIT segment. -const Trie = struct { - const Node = struct { - const Edge = struct { - from: *Node, - to: *Node, - label: []const u8, - - pub fn deinit(self: *Edge, alloc: *Allocator) void { - self.to.deinit(alloc); - alloc.destroy(self.to); - self.from = undefined; - self.to = undefined; - } - }; - - export_flags: ?u64 = null, - offset: ?u64 = null, - edges: std.ArrayListUnmanaged(Edge) = .{}, - - pub fn deinit(self: *Node, alloc: *Allocator) void { - for (self.edges.items) |*edge| { - edge.deinit(alloc); - } - self.edges.deinit(alloc); - } - - pub fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node { - // Traverse all edges. - for (self.edges.items) |*edge| { - const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything. - if (match - prefix > 0) { - // If we match, we advance further down the trie. - return edge.to.put(alloc, edge, match, label); - } - } - - if (fromEdge) |from| { - if (mem.eql(u8, from.label, label[0..prefix])) { - if (prefix == label.len) return self; - } else { - // Fixup nodes. We need to insert an intermediate node between - // from.to and self. - const mid = try alloc.create(Node); - mid.* = .{}; - const to_label = from.label; - from.to = mid; - from.label = label[0..prefix]; - - try mid.edges.append(alloc, .{ - .from = mid, - .to = self, - .label = to_label, - }); - - if (prefix == label.len) return self; // We're done. - - const new_node = try alloc.create(Node); - new_node.* = .{}; - - try mid.edges.append(alloc, .{ - .from = mid, - .to = new_node, - .label = label, - }); - - return new_node; - } - } - - // Add a new edge. - const node = try alloc.create(Node); - node.* = .{}; - - try self.edges.append(alloc, .{ - .from = self, - .to = node, - .label = label, - }); - - return node; - } - - pub fn write(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void { - if (self.offset) |off| { - // Terminal node info: encode export flags and vmaddr offset of this symbol. - var info_buf_len: usize = 0; - var info_buf: [@sizeOf(u64) * 2]u8 = undefined; - info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[0..], self.export_flags.?); - info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[info_buf_len..], off); - - // Encode the size of the terminal node info. - var size_buf: [@sizeOf(u64)]u8 = undefined; - const size_buf_len = try std.debug.leb.writeULEB128Mem(size_buf[0..], info_buf_len); - - // Now, write them to the output buffer. - try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len); - buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]); - buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]); - } else { - // Non-terminal node is delimited by 0 byte. - try buffer.append(alloc, 0); - } - // Write number of edges (max legal number of edges is 256). - try buffer.append(alloc, @intCast(u8, self.edges.items.len)); - - var node_offset_info: [@sizeOf(u8)]u64 = undefined; - for (self.edges.items) |edge, i| { - // Write edges labels leaving out space in-between to later populate - // with offsets to each node. - try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte - buffer.appendSliceAssumeCapacity(edge.label); - buffer.appendAssumeCapacity(0); - node_offset_info[i] = buffer.items.len; - const padding = [_]u8{0} ** @sizeOf(u64); - buffer.appendSliceAssumeCapacity(padding[0..]); - } - - for (self.edges.items) |edge, i| { - const offset = buffer.items.len; - try edge.to.write(alloc, buffer); - // We can now populate the offset to the node pointed by this edge. - var offset_buf: [@sizeOf(u64)]u8 = undefined; - const offset_buf_len = try std.debug.leb.writeULEB128Mem(offset_buf[0..], offset); - mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]); - } - } - }; - - root: Node, - - pub fn put(self: *Trie, alloc: *Allocator, word: []const u8) !*Node { - return self.root.put(alloc, null, 0, word); - } - - pub const WriteError = error{ OutOfMemory, NoSpaceLeft }; - - pub fn write(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void { - return self.root.write(alloc, buffer); - } - - pub fn deinit(self: *Trie, alloc: *Allocator) void { - self.root.deinit(alloc); - } -}; - base: File, /// Table of all load commands @@ -1541,19 +1393,21 @@ fn writeExportTrie(self: *MachO) !void { defer trie.deinit(self.base.allocator); const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment; - for (self.global_symbols.items) |symbol| { // TODO figure out if we should put all global symbols into the export trie const name = self.getString(symbol.n_strx); - const node = try trie.put(self.base.allocator, name); - node.offset = symbol.n_value - text_segment.vmaddr; - node.export_flags = 0; // TODO workout creation of export flags + assert(symbol.n_value >= text_segment.vmaddr); + try trie.put(self.base.allocator, .{ + .name = name, + .offset = symbol.n_value - text_segment.vmaddr, + .export_flags = 0, // TODO workout creation of export flags + }); } var buffer: std.ArrayListUnmanaged(u8) = .{}; defer buffer.deinit(self.base.allocator); - try trie.write(self.base.allocator, &buffer); + try trie.writeULEB128Mem(self.base.allocator, &buffer); const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo; try self.base.file.?.pwriteAll(buffer.items, dyld_info.export_off); @@ -1688,48 +1542,3 @@ fn satMul(a: anytype, b: anytype) @TypeOf(a, b) { const T = @TypeOf(a, b); return std.math.mul(T, a, b) catch std.math.maxInt(T); } - -test "Trie basic" { - const testing = @import("std").testing; - var gpa = testing.allocator; - - var trie: Trie = .{ - .root = .{}, - }; - defer trie.deinit(gpa); - - // root - testing.expect(trie.root.edges.items.len == 0); - - // root --- _st ---> node - try trie.put(gpa, "_st"); - testing.expect(trie.root.edges.items.len == 1); - testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st")); - - { - // root --- _st ---> node --- _start ---> node - try trie.put(gpa, "_start"); - testing.expect(trie.root.edges.items.len == 1); - - const nextEdge = &trie.root.edges.items[0]; - testing.expect(mem.eql(u8, nextEdge.label, "_st")); - testing.expect(nextEdge.to.edges.items.len == 1); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start")); - } - { - // root --- _ ---> node --- _st ---> node --- _start ---> node - // | - // | --- _main ---> node - try trie.put(gpa, "_main"); - testing.expect(trie.root.edges.items.len == 1); - - const nextEdge = &trie.root.edges.items[0]; - testing.expect(mem.eql(u8, nextEdge.label, "_")); - testing.expect(nextEdge.to.edges.items.len == 2); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st")); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main")); - - const nextNextEdge = &nextEdge.to.edges.items[0]; - testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start")); - } -} diff --git a/src/link/MachO/Trie.zig b/src/link/MachO/Trie.zig new file mode 100644 index 0000000000..24b06c8ba2 --- /dev/null +++ b/src/link/MachO/Trie.zig @@ -0,0 +1,259 @@ +/// Represents export trie used in MachO executables and dynamic libraries. +/// The purpose of an export trie is to encode as compactly as possible all +/// export symbols for the loader `dyld`. +/// The export trie encodes offset and other information using ULEB128 +/// encoding, and is part of the __LINKEDIT segment. +/// +/// Description from loader.h: +/// +/// The symbols exported by a dylib are encoded in a trie. This is a compact +/// representation that factors out common prefixes. It also reduces LINKEDIT pages +/// in RAM because it encodes all information (name, address, flags) in one small, +/// contiguous range. The export area is a stream of nodes. The first node sequentially +/// is the start node for the trie. +/// +/// Nodes for a symbol start with a uleb128 that is the length of the exported symbol +/// information for the string so far. If there is no exported symbol, the node starts +/// with a zero byte. If there is exported info, it follows the length. +/// +/// First is a uleb128 containing flags. Normally, it is followed by a uleb128 encoded +/// offset which is location of the content named by the symbol from the mach_header +/// for the image. If the flags is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags +/// is a uleb128 encoded library ordinal, then a zero terminated UTF8 string. If the string +/// is zero length, then the symbol is re-export from the specified dylib with the same name. +/// If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following the flags is two +/// uleb128s: the stub offset and the resolver offset. The stub is used by non-lazy pointers. +/// The resolver is used by lazy pointers and must be called to get the actual address to use. +/// +/// After the optional exported symbol information is a byte of how many edges (0-255) that +/// this node has leaving it, followed by each edge. Each edge is a zero terminated UTF8 of +/// the addition chars in the symbol, followed by a uleb128 offset for the node that edge points to. +const Trie = @This(); + +const std = @import("std"); +const mem = std.mem; +const leb = std.debug.leb; +const log = std.log.scoped(.link); +const Allocator = mem.Allocator; + +pub const Symbol = struct { + name: []const u8, + offset: u64, + export_flags: u64, +}; + +const Edge = struct { + from: *Node, + to: *Node, + label: []const u8, + + fn deinit(self: *Edge, alloc: *Allocator) void { + self.to.deinit(alloc); + alloc.destroy(self.to); + self.from = undefined; + self.to = undefined; + } +}; + +const Node = struct { + export_flags: ?u64 = null, + offset: ?u64 = null, + edges: std.ArrayListUnmanaged(Edge) = .{}, + + fn deinit(self: *Node, alloc: *Allocator) void { + for (self.edges.items) |*edge| { + edge.deinit(alloc); + } + self.edges.deinit(alloc); + } + + fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node { + // Traverse all edges. + for (self.edges.items) |*edge| { + const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything. + if (match - prefix > 0) { + // If we match, we advance further down the trie. + return edge.to.put(alloc, edge, match, label); + } + } + + if (fromEdge) |from| { + if (mem.eql(u8, from.label, label[0..prefix])) { + if (prefix == label.len) return self; + } else { + // Fixup nodes. We need to insert an intermediate node between + // from.to and self. + // Is: A -> B + // Should be: A -> C -> B + const mid = try alloc.create(Node); + mid.* = .{}; + const to_label = from.label; + from.to = mid; + from.label = label[0..prefix]; + + try mid.edges.append(alloc, .{ + .from = mid, + .to = self, + .label = to_label, + }); + + if (prefix == label.len) return self; // We're done. + + const new_node = try alloc.create(Node); + new_node.* = .{}; + + try mid.edges.append(alloc, .{ + .from = mid, + .to = new_node, + .label = label, + }); + + return new_node; + } + } + + // Add a new edge. + const node = try alloc.create(Node); + node.* = .{}; + + try self.edges.append(alloc, .{ + .from = self, + .to = node, + .label = label, + }); + + return node; + } + + fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void { + if (self.offset) |offset| { + // Terminal node info: encode export flags and vmaddr offset of this symbol. + var info_buf_len: usize = 0; + var info_buf: [@sizeOf(u64) * 2]u8 = undefined; + info_buf_len += try leb.writeULEB128Mem(info_buf[0..], self.export_flags.?); + info_buf_len += try leb.writeULEB128Mem(info_buf[info_buf_len..], offset); + + // Encode the size of the terminal node info. + var size_buf: [@sizeOf(u64)]u8 = undefined; + const size_buf_len = try leb.writeULEB128Mem(size_buf[0..], info_buf_len); + + // Now, write them to the output buffer. + try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len); + buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]); + buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]); + } else { + // Non-terminal node is delimited by 0 byte. + try buffer.append(alloc, 0); + } + // Write number of edges (max legal number of edges is 256). + try buffer.append(alloc, @intCast(u8, self.edges.items.len)); + + var node_offset_info: [@sizeOf(u8)]u64 = undefined; + for (self.edges.items) |edge, i| { + // Write edges labels leaving out space in-between to later populate + // with offsets to each node. + try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte + buffer.appendSliceAssumeCapacity(edge.label); + buffer.appendAssumeCapacity(0); + node_offset_info[i] = buffer.items.len; + const padding = [_]u8{0} ** @sizeOf(u64); + buffer.appendSliceAssumeCapacity(padding[0..]); + } + + for (self.edges.items) |edge, i| { + const offset = buffer.items.len; + try edge.to.writeULEB128Mem(alloc, buffer); + // We can now populate the offset to the node pointed by this edge. + // TODO this is not the approach taken by `ld64` which does several iterations + // to close the gap between the space encoding the offset to the node pointed + // by this edge. However, it seems that as long as we are contiguous, the padding + // introduced here should not influence the performance of `dyld`. I'm leaving + // this TODO here though as a reminder to re-investigate in the future and especially + // when we start working on dylibs in case `dyld` refuses to cooperate and/or the + // performance is noticably sufferring. + // Link to official impl: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp + var offset_buf: [@sizeOf(u64)]u8 = undefined; + const offset_buf_len = try leb.writeULEB128Mem(offset_buf[0..], offset); + mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]); + } + } +}; + +root: Node, + +/// Insert a symbol into the trie, updating the prefixes in the process. +/// This operation may change the layout of the trie by splicing edges in +/// certain circumstances. +pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void { + const node = try self.root.put(alloc, null, 0, symbol.name); + node.offset = symbol.offset; + node.export_flags = symbol.export_flags; +} + +pub const WriteError = error{ OutOfMemory, NoSpaceLeft }; + +/// Write the trie to a buffer ULEB128 encoded. +pub fn writeULEB128Mem(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void { + return self.root.writeULEB128Mem(alloc, buffer); +} + +pub fn deinit(self: *Trie, alloc: *Allocator) void { + self.root.deinit(alloc); +} + +test "Trie basic" { + const testing = @import("std").testing; + var gpa = testing.allocator; + + var trie: Trie = .{ + .root = .{}, + }; + defer trie.deinit(gpa); + + // root + testing.expect(trie.root.edges.items.len == 0); + + // root --- _st ---> node + try trie.put(gpa, .{ + .name = "_st", + .offset = 0, + .export_flags = 0, + }); + testing.expect(trie.root.edges.items.len == 1); + testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st")); + + { + // root --- _st ---> node --- _start ---> node + try trie.put(gpa, .{ + .name = "_start", + .offset = 0, + .export_flags = 0, + }); + testing.expect(trie.root.edges.items.len == 1); + + const nextEdge = &trie.root.edges.items[0]; + testing.expect(mem.eql(u8, nextEdge.label, "_st")); + testing.expect(nextEdge.to.edges.items.len == 1); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start")); + } + { + // root --- _ ---> node --- _st ---> node --- _start ---> node + // | + // | --- _main ---> node + try trie.put(gpa, .{ + .name = "_main", + .offset = 0, + .export_flags = 0, + }); + testing.expect(trie.root.edges.items.len == 1); + + const nextEdge = &trie.root.edges.items[0]; + testing.expect(mem.eql(u8, nextEdge.label, "_")); + testing.expect(nextEdge.to.edges.items.len == 2); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st")); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main")); + + const nextNextEdge = &nextEdge.to.edges.items[0]; + testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start")); + } +} From b5b25d38a8fa4e66e54ff1279c1becee877793f6 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 7 Oct 2020 20:32:02 +0200 Subject: [PATCH 10/22] Fix improper reuse of global symbols in MachO Signed-off-by: Jakub Konka --- src/Module.zig | 15 ++++++++++++--- src/link.zig | 8 ++++++++ src/link/Elf.zig | 4 ++-- src/link/MachO.zig | 24 ++++++++++++++++++++---- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/Module.zig b/src/Module.zig index 75b6afffcd..0c2a38be28 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -93,7 +93,7 @@ pub const Export = struct { /// Byte offset into the file that contains the export directive. src: usize, /// Represents the position of the export, if any, in the output file. - link: link.File.Elf.Export, + link: link.File.Export, /// The Decl that performs the export. Note that this is *not* the Decl being exported. owner_decl: *Decl, /// The Decl being exported. Note this is *not* the Decl performing the export. @@ -1712,7 +1712,10 @@ fn deleteDeclExports(self: *Module, decl: *Decl) void { } } if (self.comp.bin_file.cast(link.File.Elf)) |elf| { - elf.deleteExport(exp.link); + elf.deleteExport(exp.link.elf); + } + if (self.comp.bin_file.cast(link.File.MachO)) |macho| { + macho.deleteExport(exp.link.macho); } if (self.failed_exports.remove(exp)) |entry| { entry.value.destroy(self.gpa); @@ -1875,7 +1878,13 @@ pub fn analyzeExport(self: *Module, scope: *Scope, src: usize, borrowed_symbol_n new_export.* = .{ .options = .{ .name = symbol_name }, .src = src, - .link = .{}, + .link = switch (self.comp.bin_file.tag) { + .coff => .{ .coff = {} }, + .elf => .{ .elf = link.File.Elf.Export{} }, + .macho => .{ .macho = link.File.MachO.Export{} }, + .c => .{ .c = {} }, + .wasm => .{ .wasm = {} }, + }, .owner_decl = owner_decl, .exported_decl = exported_decl, .status = .in_progress, diff --git a/src/link.zig b/src/link.zig index 139977b3e2..99bca45fbe 100644 --- a/src/link.zig +++ b/src/link.zig @@ -133,6 +133,14 @@ pub const File = struct { wasm: ?Wasm.FnData, }; + pub const Export = union { + elf: Elf.Export, + coff: void, + macho: MachO.Export, + c: void, + wasm: void, + }; + /// For DWARF .debug_info. pub const DbgInfoTypeRelocsTable = std.HashMapUnmanaged(Type, DbgInfoTypeReloc, Type.hash, Type.eql, std.hash_map.DefaultMaxLoadPercentage); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index c62bb29f78..a316a9c19e 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2588,7 +2588,7 @@ pub fn updateDeclExports( }, }; const stt_bits: u8 = @truncate(u4, decl_sym.st_info); - if (exp.link.sym_index) |i| { + if (exp.link.elf.sym_index) |i| { const sym = &self.global_symbols.items[i]; sym.* = .{ .st_name = try self.updateString(sym.st_name, exp.options.name), @@ -2613,7 +2613,7 @@ pub fn updateDeclExports( .st_size = decl_sym.st_size, }; - exp.link.sym_index = @intCast(u32, i); + exp.link.elf.sym_index = @intCast(u32, i); } } } diff --git a/src/link/MachO.zig b/src/link/MachO.zig index afc54f8f7b..486620804b 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -115,6 +115,9 @@ local_symbols: std.ArrayListUnmanaged(macho.nlist_64) = .{}, global_symbols: std.ArrayListUnmanaged(macho.nlist_64) = .{}, /// Table of all undefined symbols undef_symbols: std.ArrayListUnmanaged(macho.nlist_64) = .{}, + +global_symbol_free_list: std.ArrayListUnmanaged(u32) = .{}, + dyld_stub_binder_index: ?u16 = null, /// Table of symbol names aka the string table. @@ -178,6 +181,10 @@ pub const TextBlock = struct { }; }; +pub const Export = struct { + sym_index: ?u32 = null, +}; + pub const SrcFn = struct { pub const empty = SrcFn{}; }; @@ -713,6 +720,7 @@ pub fn deinit(self: *MachO) void { self.string_table.deinit(self.base.allocator); self.undef_symbols.deinit(self.base.allocator); self.global_symbols.deinit(self.base.allocator); + self.global_symbol_free_list.deinit(self.base.allocator); self.local_symbols.deinit(self.base.allocator); self.sections.deinit(self.base.allocator); self.load_commands.deinit(self.base.allocator); @@ -837,7 +845,7 @@ pub fn updateDeclExports( }, }; const n_type = decl_sym.n_type | macho.N_EXT; - if (exp.link.sym_index) |i| { + if (exp.link.macho.sym_index) |i| { const sym = &self.global_symbols.items[i]; sym.* = .{ .n_strx = try self.updateString(sym.n_strx, exp.options.name), @@ -848,8 +856,10 @@ pub fn updateDeclExports( }; } else { const name_str_index = try self.makeString(exp.options.name); - _ = self.global_symbols.addOneAssumeCapacity(); - const i = self.global_symbols.items.len - 1; + const i = if (self.global_symbol_free_list.popOrNull()) |i| i else blk: { + _ = self.global_symbols.addOneAssumeCapacity(); + break :blk self.global_symbols.items.len - 1; + }; self.global_symbols.items[i] = .{ .n_strx = name_str_index, .n_type = n_type, @@ -858,11 +868,17 @@ pub fn updateDeclExports( .n_value = decl_sym.n_value, }; - exp.link.sym_index = @intCast(u32, i); + exp.link.macho.sym_index = @intCast(u32, i); } } } +pub fn deleteExport(self: *MachO, exp: Export) void { + const sym_index = exp.sym_index orelse return; + self.global_symbol_free_list.append(self.base.allocator, sym_index) catch {}; + self.global_symbols.items[sym_index].n_type = 0; +} + pub fn freeDecl(self: *MachO, decl: *Module.Decl) void {} pub fn getDeclVAddr(self: *MachO, decl: *const Module.Decl) u64 { From ea44d12d1be8eb17a1555f6ab794621da0212171 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 7 Oct 2020 21:19:45 +0200 Subject: [PATCH 11/22] Add writeULEB128Mem test and couple fixes Signed-off-by: Jakub Konka --- src/link/MachO/Trie.zig | 177 ++++++++++++++++++++++++++++------------ 1 file changed, 127 insertions(+), 50 deletions(-) diff --git a/src/link/MachO/Trie.zig b/src/link/MachO/Trie.zig index 24b06c8ba2..f7d37315cf 100644 --- a/src/link/MachO/Trie.zig +++ b/src/link/MachO/Trie.zig @@ -34,6 +34,7 @@ const std = @import("std"); const mem = std.mem; const leb = std.debug.leb; const log = std.log.scoped(.link); +const testing = std.testing; const Allocator = mem.Allocator; pub const Symbol = struct { @@ -67,48 +68,33 @@ const Node = struct { self.edges.deinit(alloc); } - fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node { - // Traverse all edges. + fn put(self: *Node, alloc: *Allocator, label: []const u8) !*Node { + // Check for match with edges from this node. for (self.edges.items) |*edge| { - const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything. - if (match - prefix > 0) { - // If we match, we advance further down the trie. - return edge.to.put(alloc, edge, match, label); - } - } + const match = mem.indexOfDiff(u8, edge.label, label) orelse return edge.to; + if (match == 0) continue; + if (match == edge.label.len) return edge.to.put(alloc, label[match..]); - if (fromEdge) |from| { - if (mem.eql(u8, from.label, label[0..prefix])) { - if (prefix == label.len) return self; + // Found a match, need to splice up nodes. + // From: A -> B + // To: A -> C -> B + const mid = try alloc.create(Node); + mid.* = .{}; + const to_label = edge.label; + const to_node = edge.to; + edge.to = mid; + edge.label = label[0..match]; + + try mid.edges.append(alloc, .{ + .from = mid, + .to = to_node, + .label = to_label[match..], + }); + + if (match == label.len) { + return to_node; } else { - // Fixup nodes. We need to insert an intermediate node between - // from.to and self. - // Is: A -> B - // Should be: A -> C -> B - const mid = try alloc.create(Node); - mid.* = .{}; - const to_label = from.label; - from.to = mid; - from.label = label[0..prefix]; - - try mid.edges.append(alloc, .{ - .from = mid, - .to = self, - .label = to_label, - }); - - if (prefix == label.len) return self; // We're done. - - const new_node = try alloc.create(Node); - new_node.* = .{}; - - try mid.edges.append(alloc, .{ - .from = mid, - .to = new_node, - .label = label, - }); - - return new_node; + return mid.put(alloc, label[match..]); } } @@ -148,7 +134,7 @@ const Node = struct { // Write number of edges (max legal number of edges is 256). try buffer.append(alloc, @intCast(u8, self.edges.items.len)); - var node_offset_info: [@sizeOf(u8)]u64 = undefined; + var node_offset_info: [std.math.maxInt(u8)]u64 = undefined; for (self.edges.items) |edge, i| { // Write edges labels leaving out space in-between to later populate // with offsets to each node. @@ -185,7 +171,7 @@ root: Node, /// This operation may change the layout of the trie by splicing edges in /// certain circumstances. pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void { - const node = try self.root.put(alloc, null, 0, symbol.name); + const node = try self.root.put(alloc, symbol.name); node.offset = symbol.offset; node.export_flags = symbol.export_flags; } @@ -202,9 +188,7 @@ pub fn deinit(self: *Trie, alloc: *Allocator) void { } test "Trie basic" { - const testing = @import("std").testing; var gpa = testing.allocator; - var trie: Trie = .{ .root = .{}, }; @@ -223,7 +207,7 @@ test "Trie basic" { testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st")); { - // root --- _st ---> node --- _start ---> node + // root --- _st ---> node --- art ---> node try trie.put(gpa, .{ .name = "_start", .offset = 0, @@ -234,12 +218,12 @@ test "Trie basic" { const nextEdge = &trie.root.edges.items[0]; testing.expect(mem.eql(u8, nextEdge.label, "_st")); testing.expect(nextEdge.to.edges.items.len == 1); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start")); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "art")); } { - // root --- _ ---> node --- _st ---> node --- _start ---> node + // root --- _ ---> node --- st ---> node --- art ---> node // | - // | --- _main ---> node + // | --- main ---> node try trie.put(gpa, .{ .name = "_main", .offset = 0, @@ -250,10 +234,103 @@ test "Trie basic" { const nextEdge = &trie.root.edges.items[0]; testing.expect(mem.eql(u8, nextEdge.label, "_")); testing.expect(nextEdge.to.edges.items.len == 2); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st")); - testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main")); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "st")); + testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "main")); const nextNextEdge = &nextEdge.to.edges.items[0]; - testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start")); + testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "art")); } } + +test "Trie.writeULEB128Mem" { + var gpa = testing.allocator; + var trie: Trie = .{ + .root = .{}, + }; + defer trie.deinit(gpa); + + try trie.put(gpa, .{ + .name = "__mh_execute_header", + .offset = 0, + .export_flags = 0, + }); + try trie.put(gpa, .{ + .name = "_main", + .offset = 0x1000, + .export_flags = 0, + }); + + var buffer: std.ArrayListUnmanaged(u8) = .{}; + defer buffer.deinit(gpa); + + try trie.writeULEB128Mem(gpa, &buffer); + + const exp_buffer = [_]u8{ + 0x0, + 0x1, + 0x5f, + 0x0, + 0xc, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x2, + 0x5f, + 0x6d, + 0x68, + 0x5f, + 0x65, + 0x78, + 0x65, + 0x63, + 0x75, + 0x74, + 0x65, + 0x5f, + 0x68, + 0x65, + 0x61, + 0x64, + 0x65, + 0x72, + 0x0, + 0x36, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x6d, + 0x61, + 0x69, + 0x6e, + 0x0, + 0x3a, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x2, + 0x0, + 0x0, + 0x0, + 0x3, + 0x0, + 0x80, + 0x20, + 0x0, + }; + + testing.expect(buffer.items.len == exp_buffer.len); + testing.expect(mem.eql(u8, buffer.items, exp_buffer[0..])); +} From 5f86505cf79a0ce75e1a02602ae0e9c845024982 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Oct 2020 17:52:08 +0200 Subject: [PATCH 12/22] Fix ULEB128 encoding of trie Use algorithm described in official Apple `ld64` implementation. Link: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp Signed-off-by: Jakub Konka --- src/link/MachO.zig | 2 +- src/link/MachO/Trie.zig | 144 +++++++++++++++++++++++----------------- 2 files changed, 84 insertions(+), 62 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 486620804b..ce9b7d1706 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -1415,7 +1415,7 @@ fn writeExportTrie(self: *MachO) !void { assert(symbol.n_value >= text_segment.vmaddr); try trie.put(self.base.allocator, .{ .name = name, - .offset = symbol.n_value - text_segment.vmaddr, + .vmaddr_offset = symbol.n_value - text_segment.vmaddr, .export_flags = 0, // TODO workout creation of export flags }); } diff --git a/src/link/MachO/Trie.zig b/src/link/MachO/Trie.zig index f7d37315cf..4c13262d2d 100644 --- a/src/link/MachO/Trie.zig +++ b/src/link/MachO/Trie.zig @@ -39,7 +39,7 @@ const Allocator = mem.Allocator; pub const Symbol = struct { name: []const u8, - offset: u64, + vmaddr_offset: u64, export_flags: u64, }; @@ -58,7 +58,8 @@ const Edge = struct { const Node = struct { export_flags: ?u64 = null, - offset: ?u64 = null, + vmaddr_offset: ?u64 = null, + trie_offset: usize = 0, edges: std.ArrayListUnmanaged(Edge) = .{}, fn deinit(self: *Node, alloc: *Allocator) void { @@ -111,8 +112,8 @@ const Node = struct { return node; } - fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void { - if (self.offset) |offset| { + fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) !void { + if (self.vmaddr_offset) |offset| { // Terminal node info: encode export flags and vmaddr offset of this symbol. var info_buf_len: usize = 0; var info_buf: [@sizeOf(u64) * 2]u8 = undefined; @@ -134,34 +135,53 @@ const Node = struct { // Write number of edges (max legal number of edges is 256). try buffer.append(alloc, @intCast(u8, self.edges.items.len)); - var node_offset_info: [std.math.maxInt(u8)]u64 = undefined; - for (self.edges.items) |edge, i| { - // Write edges labels leaving out space in-between to later populate - // with offsets to each node. - try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte + for (self.edges.items) |edge| { + // Write edges labels. + try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1); // +1 to account for null-byte buffer.appendSliceAssumeCapacity(edge.label); buffer.appendAssumeCapacity(0); - node_offset_info[i] = buffer.items.len; - const padding = [_]u8{0} ** @sizeOf(u64); - buffer.appendSliceAssumeCapacity(padding[0..]); + + var buf: [@sizeOf(u64)]u8 = undefined; + const buf_len = try leb.writeULEB128Mem(buf[0..], edge.to.trie_offset); + try buffer.appendSlice(alloc, buf[0..buf_len]); + } + } + + const UpdateResult = struct { + node_size: usize, + updated: bool, + }; + + fn updateOffset(self: *Node, offset: usize) UpdateResult { + var node_size: usize = 0; + if (self.vmaddr_offset) |vmaddr| { + node_size += sizeULEB128Mem(self.export_flags.?); + node_size += sizeULEB128Mem(vmaddr); + node_size += sizeULEB128Mem(node_size); + } else { + node_size += 1; // 0x0 for non-terminal nodes + } + node_size += 1; // 1 byte for edge count + + for (self.edges.items) |edge| { + node_size += edge.label.len + 1 + sizeULEB128Mem(edge.to.trie_offset); } - for (self.edges.items) |edge, i| { - const offset = buffer.items.len; - try edge.to.writeULEB128Mem(alloc, buffer); - // We can now populate the offset to the node pointed by this edge. - // TODO this is not the approach taken by `ld64` which does several iterations - // to close the gap between the space encoding the offset to the node pointed - // by this edge. However, it seems that as long as we are contiguous, the padding - // introduced here should not influence the performance of `dyld`. I'm leaving - // this TODO here though as a reminder to re-investigate in the future and especially - // when we start working on dylibs in case `dyld` refuses to cooperate and/or the - // performance is noticably sufferring. - // Link to official impl: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp - var offset_buf: [@sizeOf(u64)]u8 = undefined; - const offset_buf_len = try leb.writeULEB128Mem(offset_buf[0..], offset); - mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]); + const updated = offset != self.trie_offset; + self.trie_offset = offset; + + return .{ .node_size = node_size, .updated = updated }; + } + + fn sizeULEB128Mem(value: u64) usize { + var res: usize = 0; + var v = value; + while (true) { + v = v >> 7; + res += 1; + if (v == 0) break; } + return res; } }; @@ -172,15 +192,38 @@ root: Node, /// certain circumstances. pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void { const node = try self.root.put(alloc, symbol.name); - node.offset = symbol.offset; + node.vmaddr_offset = symbol.vmaddr_offset; node.export_flags = symbol.export_flags; } -pub const WriteError = error{ OutOfMemory, NoSpaceLeft }; - /// Write the trie to a buffer ULEB128 encoded. -pub fn writeULEB128Mem(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void { - return self.root.writeULEB128Mem(alloc, buffer); +pub fn writeULEB128Mem(self: *Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) !void { + var ordered_nodes: std.ArrayListUnmanaged(*Node) = .{}; + defer ordered_nodes.deinit(alloc); + + try walkInOrder(&self.root, alloc, &ordered_nodes); + + var more: bool = true; + while (more) { + var offset: usize = 0; + more = false; + for (ordered_nodes.items) |node| { + const res = node.updateOffset(offset); + offset += res.node_size; + if (res.updated) more = true; + } + } + + for (ordered_nodes.items) |node| { + try node.writeULEB128Mem(alloc, buffer); + } +} + +fn walkInOrder(node: *Node, alloc: *Allocator, list: *std.ArrayListUnmanaged(*Node)) error{OutOfMemory}!void { + try list.append(alloc, node); + for (node.edges.items) |*edge| { + try walkInOrder(edge.to, alloc, list); + } } pub fn deinit(self: *Trie, alloc: *Allocator) void { @@ -200,7 +243,7 @@ test "Trie basic" { // root --- _st ---> node try trie.put(gpa, .{ .name = "_st", - .offset = 0, + .vmaddr_offset = 0, .export_flags = 0, }); testing.expect(trie.root.edges.items.len == 1); @@ -210,7 +253,7 @@ test "Trie basic" { // root --- _st ---> node --- art ---> node try trie.put(gpa, .{ .name = "_start", - .offset = 0, + .vmaddr_offset = 0, .export_flags = 0, }); testing.expect(trie.root.edges.items.len == 1); @@ -226,7 +269,7 @@ test "Trie basic" { // | --- main ---> node try trie.put(gpa, .{ .name = "_main", - .offset = 0, + .vmaddr_offset = 0, .export_flags = 0, }); testing.expect(trie.root.edges.items.len == 1); @@ -251,12 +294,12 @@ test "Trie.writeULEB128Mem" { try trie.put(gpa, .{ .name = "__mh_execute_header", - .offset = 0, + .vmaddr_offset = 0, .export_flags = 0, }); try trie.put(gpa, .{ .name = "_main", - .offset = 0x1000, + .vmaddr_offset = 0x1000, .export_flags = 0, }); @@ -270,14 +313,7 @@ test "Trie.writeULEB128Mem" { 0x1, 0x5f, 0x0, - 0xc, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, + 0x5, 0x0, 0x2, 0x5f, @@ -299,27 +335,13 @@ test "Trie.writeULEB128Mem" { 0x65, 0x72, 0x0, - 0x36, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, + 0x21, 0x6d, 0x61, 0x69, 0x6e, 0x0, - 0x3a, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, - 0x0, + 0x25, 0x2, 0x0, 0x0, From ba41e599bfaff2c614c4edfbe5c7ffe94b437486 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Oct 2020 18:10:32 +0200 Subject: [PATCH 13/22] Clean up writing the trie into ULEB128 byte stream Prealloc as much as possible to improve alloc performance. Signed-off-by: Jakub Konka --- src/link/MachO.zig | 4 +- src/link/MachO/Trie.zig | 144 +++++++++++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 36 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index ce9b7d1706..697e4f0be3 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -1403,9 +1403,7 @@ fn writeAllUndefSymbols(self: *MachO) !void { fn writeExportTrie(self: *MachO) !void { if (self.global_symbols.items.len == 0) return; // No exports, nothing to do. - var trie: Trie = .{ - .root = .{}, - }; + var trie: Trie = .{}; defer trie.deinit(self.base.allocator); const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment; diff --git a/src/link/MachO/Trie.zig b/src/link/MachO/Trie.zig index 4c13262d2d..d19914f292 100644 --- a/src/link/MachO/Trie.zig +++ b/src/link/MachO/Trie.zig @@ -35,6 +35,7 @@ const mem = std.mem; const leb = std.debug.leb; const log = std.log.scoped(.link); const testing = std.testing; +const assert = std.debug.assert; const Allocator = mem.Allocator; pub const Symbol = struct { @@ -57,9 +58,13 @@ const Edge = struct { }; const Node = struct { + /// Export flags associated with this exported symbol (if any). export_flags: ?u64 = null, + /// VM address offset wrt to the section this symbol is defined against (if any). vmaddr_offset: ?u64 = null, - trie_offset: usize = 0, + /// Offset of this node in the trie output byte stream. + trie_offset: ?usize = null, + /// List of all edges originating from this node. edges: std.ArrayListUnmanaged(Edge) = .{}, fn deinit(self: *Node, alloc: *Allocator) void { @@ -69,12 +74,24 @@ const Node = struct { self.edges.deinit(alloc); } - fn put(self: *Node, alloc: *Allocator, label: []const u8) !*Node { + const PutResult = struct { + /// Node reached at this stage of `put` op. + node: *Node, + /// Count of newly inserted nodes at this stage of `put` op. + node_count: usize, + }; + + /// Inserts a new node starting from `self`. + fn put(self: *Node, alloc: *Allocator, label: []const u8, node_count: usize) !PutResult { + var curr_node_count = node_count; // Check for match with edges from this node. for (self.edges.items) |*edge| { - const match = mem.indexOfDiff(u8, edge.label, label) orelse return edge.to; + const match = mem.indexOfDiff(u8, edge.label, label) orelse return PutResult{ + .node = edge.to, + .node_count = curr_node_count, + }; if (match == 0) continue; - if (match == edge.label.len) return edge.to.put(alloc, label[match..]); + if (match == edge.label.len) return edge.to.put(alloc, label[match..], curr_node_count); // Found a match, need to splice up nodes. // From: A -> B @@ -85,6 +102,7 @@ const Node = struct { const to_node = edge.to; edge.to = mid; edge.label = label[0..match]; + curr_node_count += 1; try mid.edges.append(alloc, .{ .from = mid, @@ -93,15 +111,16 @@ const Node = struct { }); if (match == label.len) { - return to_node; + return PutResult{ .node = to_node, .node_count = curr_node_count }; } else { - return mid.put(alloc, label[match..]); + return mid.put(alloc, label[match..], curr_node_count); } } - // Add a new edge. + // Add a new node. const node = try alloc.create(Node); node.* = .{}; + curr_node_count += 1; try self.edges.append(alloc, .{ .from = self, @@ -109,10 +128,13 @@ const Node = struct { .label = label, }); - return node; + return PutResult{ .node = node, .node_count = curr_node_count }; } - fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) !void { + /// This method should only be called *after* updateOffset has been called! + /// In case this is not upheld, this method will panic. + fn writeULEB128Mem(self: Node, buffer: *std.ArrayListUnmanaged(u8)) !void { + assert(self.trie_offset != null); // You need to call updateOffset first. if (self.vmaddr_offset) |offset| { // Terminal node info: encode export flags and vmaddr offset of this symbol. var info_buf_len: usize = 0; @@ -125,33 +147,35 @@ const Node = struct { const size_buf_len = try leb.writeULEB128Mem(size_buf[0..], info_buf_len); // Now, write them to the output buffer. - try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len); buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]); buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]); } else { // Non-terminal node is delimited by 0 byte. - try buffer.append(alloc, 0); + buffer.appendAssumeCapacity(0); } // Write number of edges (max legal number of edges is 256). - try buffer.append(alloc, @intCast(u8, self.edges.items.len)); + buffer.appendAssumeCapacity(@intCast(u8, self.edges.items.len)); for (self.edges.items) |edge| { // Write edges labels. - try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1); // +1 to account for null-byte buffer.appendSliceAssumeCapacity(edge.label); buffer.appendAssumeCapacity(0); var buf: [@sizeOf(u64)]u8 = undefined; - const buf_len = try leb.writeULEB128Mem(buf[0..], edge.to.trie_offset); - try buffer.appendSlice(alloc, buf[0..buf_len]); + const buf_len = try leb.writeULEB128Mem(buf[0..], edge.to.trie_offset.?); + buffer.appendSliceAssumeCapacity(buf[0..buf_len]); } } const UpdateResult = struct { + /// Current size of this node in bytes. node_size: usize, + /// True if the trie offset of this node in the output byte stream + /// would need updating; false otherwise. updated: bool, }; + /// Updates offset of this node in the output byte stream. fn updateOffset(self: *Node, offset: usize) UpdateResult { var node_size: usize = 0; if (self.vmaddr_offset) |vmaddr| { @@ -164,15 +188,18 @@ const Node = struct { node_size += 1; // 1 byte for edge count for (self.edges.items) |edge| { - node_size += edge.label.len + 1 + sizeULEB128Mem(edge.to.trie_offset); + const next_node_offset = edge.to.trie_offset orelse 0; + node_size += edge.label.len + 1 + sizeULEB128Mem(next_node_offset); } - const updated = offset != self.trie_offset; + const trie_offset = self.trie_offset orelse 0; + const updated = offset != trie_offset; self.trie_offset = offset; return .{ .node_size = node_size, .updated = updated }; } + /// Calculates number of bytes in ULEB128 encoding of value. fn sizeULEB128Mem(value: u64) usize { var res: usize = 0; var v = value; @@ -185,15 +212,22 @@ const Node = struct { } }; -root: Node, +/// Count of nodes in the trie. +/// The count is updated at every `put` call. +/// The trie always consists of at least a root node, hence +/// the count always starts at 1. +node_count: usize = 1, +/// The root node of the trie. +root: Node = .{}, /// Insert a symbol into the trie, updating the prefixes in the process. /// This operation may change the layout of the trie by splicing edges in /// certain circumstances. pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void { - const node = try self.root.put(alloc, symbol.name); - node.vmaddr_offset = symbol.vmaddr_offset; - node.export_flags = symbol.export_flags; + const res = try self.root.put(alloc, symbol.name, 0); + self.node_count += res.node_count; + res.node.vmaddr_offset = symbol.vmaddr_offset; + res.node.export_flags = symbol.export_flags; } /// Write the trie to a buffer ULEB128 encoded. @@ -201,11 +235,13 @@ pub fn writeULEB128Mem(self: *Trie, alloc: *Allocator, buffer: *std.ArrayListUnm var ordered_nodes: std.ArrayListUnmanaged(*Node) = .{}; defer ordered_nodes.deinit(alloc); - try walkInOrder(&self.root, alloc, &ordered_nodes); + try ordered_nodes.ensureCapacity(alloc, self.node_count); + walkInOrder(&self.root, &ordered_nodes); + var offset: usize = 0; var more: bool = true; while (more) { - var offset: usize = 0; + offset = 0; more = false; for (ordered_nodes.items) |node| { const res = node.updateOffset(offset); @@ -214,15 +250,17 @@ pub fn writeULEB128Mem(self: *Trie, alloc: *Allocator, buffer: *std.ArrayListUnm } } + try buffer.ensureCapacity(alloc, buffer.items.len + offset); for (ordered_nodes.items) |node| { - try node.writeULEB128Mem(alloc, buffer); + try node.writeULEB128Mem(buffer); } } -fn walkInOrder(node: *Node, alloc: *Allocator, list: *std.ArrayListUnmanaged(*Node)) error{OutOfMemory}!void { - try list.append(alloc, node); +/// Walks the trie in DFS order gathering all nodes into a linear stream of nodes. +fn walkInOrder(node: *Node, list: *std.ArrayListUnmanaged(*Node)) void { + list.appendAssumeCapacity(node); for (node.edges.items) |*edge| { - try walkInOrder(edge.to, alloc, list); + walkInOrder(edge.to, list); } } @@ -230,11 +268,53 @@ pub fn deinit(self: *Trie, alloc: *Allocator) void { self.root.deinit(alloc); } +test "Trie node count" { + var gpa = testing.allocator; + var trie: Trie = .{}; + defer trie.deinit(gpa); + + testing.expectEqual(trie.node_count, 1); + + try trie.put(gpa, .{ + .name = "_main", + .vmaddr_offset = 0, + .export_flags = 0, + }); + testing.expectEqual(trie.node_count, 2); + + // Inserting the same node shouldn't update the trie. + try trie.put(gpa, .{ + .name = "_main", + .vmaddr_offset = 0, + .export_flags = 0, + }); + testing.expectEqual(trie.node_count, 2); + + try trie.put(gpa, .{ + .name = "__mh_execute_header", + .vmaddr_offset = 0x1000, + .export_flags = 0, + }); + testing.expectEqual(trie.node_count, 4); + + // Inserting the same node shouldn't update the trie. + try trie.put(gpa, .{ + .name = "__mh_execute_header", + .vmaddr_offset = 0x1000, + .export_flags = 0, + }); + testing.expectEqual(trie.node_count, 4); + try trie.put(gpa, .{ + .name = "_main", + .vmaddr_offset = 0, + .export_flags = 0, + }); + testing.expectEqual(trie.node_count, 4); +} + test "Trie basic" { var gpa = testing.allocator; - var trie: Trie = .{ - .root = .{}, - }; + var trie: Trie = .{}; defer trie.deinit(gpa); // root @@ -287,9 +367,7 @@ test "Trie basic" { test "Trie.writeULEB128Mem" { var gpa = testing.allocator; - var trie: Trie = .{ - .root = .{}, - }; + var trie: Trie = .{}; defer trie.deinit(gpa); try trie.put(gpa, .{ From a4828f6d0f94039473aa3ac53e122037d2d6a2d3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Oct 2020 22:45:21 -0700 Subject: [PATCH 14/22] std.c (darwin) update to new opaque syntax This was an undetected conflict between 76a195473dac059a842fed2a6ba581ca99947d2b and 95a37373e9f576854956c2909cc128b5b6388ec6 --- lib/std/c/darwin.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig index a3482c73f6..cc7e6d0463 100644 --- a/lib/std/c/darwin.zig +++ b/lib/std/c/darwin.zig @@ -23,7 +23,7 @@ pub const COPYFILE_STAT = 1 << 1; pub const COPYFILE_XATTR = 1 << 2; pub const COPYFILE_DATA = 1 << 3; -pub const copyfile_state_t = *@Type(.Opaque); +pub const copyfile_state_t = *opaque {}; pub extern "c" fn fcopyfile(from: fd_t, to: fd_t, state: ?copyfile_state_t, flags: u32) c_int; pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8; From 1951ecb228d5eb9956ef4487e20be03d87a2153d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Oct 2020 22:48:16 -0700 Subject: [PATCH 15/22] add a code of conduct this has always been the rules, they are just written down now. also link to ziglearn.org in the readme --- CODE_OF_CONDUCT.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 77 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f0bded8356 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,75 @@ +# Code of Conduct + +Hello, and welcome! 👋 + +The Zig community is decentralized. Anyone is free to start and maintain their +own space for people to gather, and edit +[the Community wiki page](https://github.com/ziglang/zig/wiki/Community) to add +a link. There is no concept of "official" or "unofficial", however, each +gathering place has its own moderators and rules. + +This is Andrew Kelley speaking. At least for now, I'm the moderator of the +ziglang organization GitHub repositories and the #zig IRC channel on Freenode. +**This document contains the rules that govern these two spaces only**. + +The rules here are strict. This space is for focused, on topic, technical work +on the Zig project only. It is everyone's responsibility to maintain a positive +environment, especially when disagreements occur. + +## Our Standards + +Examples of behavior that contribute to creating a positive environment include: + + * Using welcoming and inclusive language. + * Being respectful of differing viewpoints and experiences. + * Gracefully accepting constructive criticism. + * Helping another person accomplish their own goals. + * Showing empathy towards others. + * Showing appreciation for others' work. + * Validating someone else's experience, skills, insight, and use cases. + +Examples of unacceptable behavior by participants include: + + * Unwelcome sexual attention or advances, or use of sexualized language or + imagery that causes discomfort. + * Trolling, insulting/derogatory comments, and personal attacks. Anything + antagonistic towards someone else. + * Off-topic discussion of any kind - especially offensive or sensitive issues. + * Publishing others' private information, such as a physical or electronic + address, without explicit permission. + * Discussing this Code of Conduct or publicly accusing someone of violating it. + * Making someone else feel like an outsider or implying a lack of technical + abilities. + * Destructive behavior. Anything that harms Zig or another open-source project. + +## Enforcement + +If you need to report an issue you can contact me or Loris Cro, who are both +paid by the Zig Software Foundation, and so moderation of this space is part of +our job. We will swiftly remove anyone who is antagonizing others or being +generally destructive. + +This includes Private Harassment. If person A is directly harassed or +antagonized by person B, person B will be blocked from participating in this +space even if the harassment didn't take place on one of the mediums directly +under rule of this Code of Conduct. + +As noted, discussing this Code of Conduct should not take place on GitHub or IRC +because these spaces are for directly working on code, not for meta-discussion. +If you have any issues with it, you can contact me directly, or you can join one +of the community spaces that has different rules. + + * Andrew Kelley + * Loris Cro + +## Conclusion + +Thanks for reading the rules. Together, we can make this space welcoming and +inclusive for everyone, regardless of age, body size, disability, ethnicity, +sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, +religion, or sexual identity and orientation. + +Sincerely, + +Andrew ✌️ diff --git a/README.md b/README.md index 15bf11e5f4..0bf3bf2642 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,10 @@ A general-purpose programming language and toolchain for maintaining * [Introduction](https://ziglang.org/#Introduction) * [Download & Documentation](https://ziglang.org/download) + * [Chapter 0 - Getting Started | ZigLearn.org](https://ziglearn.org/) * [Community](https://github.com/ziglang/zig/wiki/Community) * [Contributing](https://github.com/ziglang/zig/blob/master/CONTRIBUTING.md) + * [Code of Conduct](https://github.com/ziglang/zig/blob/master/CODE_OF_CONDUCT.md) * [Frequently Asked Questions](https://github.com/ziglang/zig/wiki/FAQ) * [Community Projects](https://github.com/ziglang/zig/wiki/Community-Projects) From bc6904eccc53da5b4de0728c8b52e6d0e9ed522e Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Thu, 8 Oct 2020 21:40:56 -0400 Subject: [PATCH 16/22] include compiler_rt and c for wasm static libraries --- src/Compilation.zig | 5 ++++- src/link/Wasm.zig | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 3e74df35af..e87233eb58 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -922,7 +922,10 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { try comp.work_queue.writeItem(.libcxx); try comp.work_queue.writeItem(.libcxxabi); } - if (is_exe_or_dyn_lib and build_options.is_stage1) { + + const needs_compiler_rt_and_c = is_exe_or_dyn_lib or + (comp.getTarget().isWasm() and comp.bin_file.options.output_mode != .Obj); + if (needs_compiler_rt_and_c and build_options.is_stage1) { try comp.work_queue.writeItem(.{ .libcompiler_rt = {} }); if (!comp.bin_file.options.link_libc) { try comp.work_queue.writeItem(.{ .zig_libc = {} }); diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 3f879a3b32..e4fd409a4d 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -374,7 +374,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { try argv.append(p); } - if (self.base.options.output_mode == .Exe and !self.base.options.is_compiler_rt_or_libc) { + if (self.base.options.output_mode != .Obj and !self.base.options.is_compiler_rt_or_libc) { if (!self.base.options.link_libc) { try argv.append(comp.libc_static_lib.?.full_object_path); } From 8dc40236153e7c7d1b8378a117d8453e3b262933 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 9 Oct 2020 17:22:39 +0200 Subject: [PATCH 17/22] Apply nitpick: top-level doc comments Signed-off-by: Jakub Konka --- src/link/MachO/Trie.zig | 60 ++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/link/MachO/Trie.zig b/src/link/MachO/Trie.zig index d19914f292..e077df101d 100644 --- a/src/link/MachO/Trie.zig +++ b/src/link/MachO/Trie.zig @@ -1,33 +1,33 @@ -/// Represents export trie used in MachO executables and dynamic libraries. -/// The purpose of an export trie is to encode as compactly as possible all -/// export symbols for the loader `dyld`. -/// The export trie encodes offset and other information using ULEB128 -/// encoding, and is part of the __LINKEDIT segment. -/// -/// Description from loader.h: -/// -/// The symbols exported by a dylib are encoded in a trie. This is a compact -/// representation that factors out common prefixes. It also reduces LINKEDIT pages -/// in RAM because it encodes all information (name, address, flags) in one small, -/// contiguous range. The export area is a stream of nodes. The first node sequentially -/// is the start node for the trie. -/// -/// Nodes for a symbol start with a uleb128 that is the length of the exported symbol -/// information for the string so far. If there is no exported symbol, the node starts -/// with a zero byte. If there is exported info, it follows the length. -/// -/// First is a uleb128 containing flags. Normally, it is followed by a uleb128 encoded -/// offset which is location of the content named by the symbol from the mach_header -/// for the image. If the flags is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags -/// is a uleb128 encoded library ordinal, then a zero terminated UTF8 string. If the string -/// is zero length, then the symbol is re-export from the specified dylib with the same name. -/// If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following the flags is two -/// uleb128s: the stub offset and the resolver offset. The stub is used by non-lazy pointers. -/// The resolver is used by lazy pointers and must be called to get the actual address to use. -/// -/// After the optional exported symbol information is a byte of how many edges (0-255) that -/// this node has leaving it, followed by each edge. Each edge is a zero terminated UTF8 of -/// the addition chars in the symbol, followed by a uleb128 offset for the node that edge points to. +//! Represents export trie used in MachO executables and dynamic libraries. +//! The purpose of an export trie is to encode as compactly as possible all +//! export symbols for the loader `dyld`. +//! The export trie encodes offset and other information using ULEB128 +//! encoding, and is part of the __LINKEDIT segment. +//! +//! Description from loader.h: +//! +//! The symbols exported by a dylib are encoded in a trie. This is a compact +//! representation that factors out common prefixes. It also reduces LINKEDIT pages +//! in RAM because it encodes all information (name, address, flags) in one small, +//! contiguous range. The export area is a stream of nodes. The first node sequentially +//! is the start node for the trie. +//! +//! Nodes for a symbol start with a uleb128 that is the length of the exported symbol +//! information for the string so far. If there is no exported symbol, the node starts +//! with a zero byte. If there is exported info, it follows the length. +//! +//! First is a uleb128 containing flags. Normally, it is followed by a uleb128 encoded +//! offset which is location of the content named by the symbol from the mach_header +//! for the image. If the flags is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags +//! is a uleb128 encoded library ordinal, then a zero terminated UTF8 string. If the string +//! is zero length, then the symbol is re-export from the specified dylib with the same name. +//! If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following the flags is two +//! uleb128s: the stub offset and the resolver offset. The stub is used by non-lazy pointers. +//! The resolver is used by lazy pointers and must be called to get the actual address to use. +//! +//! After the optional exported symbol information is a byte of how many edges (0-255) that +//! this node has leaving it, followed by each edge. Each edge is a zero terminated UTF8 of +//! the addition chars in the symbol, followed by a uleb128 offset for the node that edge points to. const Trie = @This(); const std = @import("std"); From 57912964af0247a7aa940f76d09a8994d8fe1ec8 Mon Sep 17 00:00:00 2001 From: mlarouche Date: Fri, 9 Oct 2020 16:50:43 -0400 Subject: [PATCH 18/22] Use regular file for caching stage 1 hash digest instead of symlink, fix zig build caching on Windows Fix #6500 --- lib/std/fs.zig | 9 +++++++++ src/Compilation.zig | 8 ++++---- src/link.zig | 8 ++++---- src/link/Coff.zig | 10 +++++----- src/link/Elf.zig | 10 +++++----- src/link/MachO.zig | 10 +++++----- src/link/Wasm.zig | 8 ++++---- 7 files changed, 36 insertions(+), 27 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 5e440ec900..bebc954746 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1490,6 +1490,15 @@ pub const Dir = struct { return os.windows.ReadLink(self.fd, sub_path_w, buffer); } + /// Read all of file contents using a preallocated buffer + pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 { + var file = try self.openFile(file_path, .{}); + defer file.close(); + + const end_index = try file.readAll(buffer); + return buffer[0..end_index]; + } + /// On success, caller owns returned buffer. /// If the file is larger than `max_bytes`, returns `error.FileTooBig`. pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 { diff --git a/src/Compilation.zig b/src/Compilation.zig index e87233eb58..0bff2372d9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2605,8 +2605,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node // We use an extra hex-encoded byte here to store some flags. var prev_digest_buf: [digest.len + 2]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("stage1 {} new_digest={} readFile error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -2792,8 +2792,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{ digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup, }); - directory.handle.symLink(&digest_plus_flags, id_symlink_basename, .{}) catch |err| { - log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)}); + directory.handle.writeFile(id_symlink_basename, &digest_plus_flags) catch |err| { + log.warn("failed to save stage1 hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. man.writeManifest() catch |err| { diff --git a/src/link.zig b/src/link.zig index 99bca45fbe..cd41c54cd4 100644 --- a/src/link.zig +++ b/src/link.zig @@ -466,8 +466,8 @@ pub const File = struct { const digest = ch.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| b: { - log.debug("archive new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| b: { + log.debug("archive new_digest={} readFile error: {}", .{ digest, @errorName(err) }); break :b prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { @@ -512,8 +512,8 @@ pub const File = struct { const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type); if (bad) return error.UnableToWriteArchive; - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save archive hash digest symlink: {}", .{@errorName(err)}); + directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + std.log.warn("failed to save archive hash digest file: {}", .{@errorName(err)}); }; ch.writeManifest() catch |err| { diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 24c3833e0a..5c7975743f 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -854,8 +854,8 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { _ = try man.hit(); digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("COFF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("COFF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -1180,10 +1180,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling symlink with the digest. If it fails we can continue; it only + // Update the dangling file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. man.writeManifest() catch |err| { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index a316a9c19e..4fcfe78901 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1326,8 +1326,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("ELF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("ELF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -1647,10 +1647,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling symlink with the digest. If it fails we can continue; it only + // Update the dangling file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. man.writeManifest() catch |err| { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 697e4f0be3..20fda846f5 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -419,8 +419,8 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("MachO LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("MachO LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -674,10 +674,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling symlink with the digest. If it fails we can continue; it only + // Update the dangling file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { - std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); + directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. man.writeManifest() catch |err| { diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index e4fd409a4d..320f90de19 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -310,8 +310,8 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("WASM LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { + log.debug("WASM LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -424,9 +424,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling symlink with the digest. If it fails we can continue; it only + // Update the dangling file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { + directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. From 9f8f4464353460825856b848dfe2480de20d8d55 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 Oct 2020 16:45:39 -0700 Subject: [PATCH 19/22] fixups to previous commit * std.fs.Dir.readFile: add doc comments to explain what it means when the returned slice has the same length as the supplied buffer. * introduce readSmallFile / writeSmallFile to abstract over the decision to use symlink or file contents to store data. --- lib/std/fs.zig | 6 +++++- src/Cache.zig | 24 ++++++++++++++++++++++++ src/Compilation.zig | 14 +++++++++----- src/link.zig | 8 ++++++-- src/link/Coff.zig | 12 ++++++++---- src/link/Elf.zig | 12 ++++++++---- src/link/MachO.zig | 12 ++++++++---- src/link/Wasm.zig | 12 ++++++++---- 8 files changed, 76 insertions(+), 24 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index bebc954746..0f8c6e0d24 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1490,7 +1490,11 @@ pub const Dir = struct { return os.windows.ReadLink(self.fd, sub_path_w, buffer); } - /// Read all of file contents using a preallocated buffer + /// Read all of file contents using a preallocated buffer. + /// The returned slice has the same pointer as `buffer`. If the length matches `buffer.len` + /// the situation is ambiguous. It could either mean that the entire file was read, and + /// it exactly fits the buffer, or it could mean the buffer was not big enough for the + /// entire file. pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 { var file = try self.openFile(file_path, .{}); defer file.close(); diff --git a/src/Cache.zig b/src/Cache.zig index dff6f7e38e..1adf72d16f 100644 --- a/src/Cache.zig +++ b/src/Cache.zig @@ -576,6 +576,30 @@ pub const Manifest = struct { } }; +/// On operating systems that support symlinks, does a readlink. On other operating systems, +/// uses the file contents. Windows supports symlinks but only with elevated privileges, so +/// it is treated as not supporting symlinks. +pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 { + if (std.Target.current.os.tag == .windows) { + return dir.readFile(sub_path, buffer); + } else { + return dir.readLink(sub_path, buffer); + } +} + +/// On operating systems that support symlinks, does a symlink. On other operating systems, +/// uses the file contents. Windows supports symlinks but only with elevated privileges, so +/// it is treated as not supporting symlinks. +/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer. +pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void { + assert(data.len <= 255); + if (std.Target.current.os.tag == .windows) { + return dir.writeFile(sub_path, data); + } else { + return dir.symLink(data, sub_path, .{}); + } +} + fn hashFile(file: fs.File, bin_digest: []u8) !void { var buf: [1024]u8 = undefined; diff --git a/src/Compilation.zig b/src/Compilation.zig index 0bff2372d9..1fddda42f9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2605,8 +2605,12 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node // We use an extra hex-encoded byte here to store some flags. var prev_digest_buf: [digest.len + 2]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("stage1 {} new_digest={} readFile error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| blk: { + log.debug("stage1 {} new_digest={} error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -2777,7 +2781,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node const digest = man.final(); - // Update the dangling symlink with the digest. If it fails we can continue; it only + // Update the small file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. const stage1_flags_byte = @bitCast(u8, mod.stage1_flags); log.debug("stage1 {} final digest={} flags={x}", .{ @@ -2792,10 +2796,10 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{ digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup, }); - directory.handle.writeFile(id_symlink_basename, &digest_plus_flags) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest_plus_flags) catch |err| { log.warn("failed to save stage1 hash digest file: {}", .{@errorName(err)}); }; - // Again failure here only means an unnecessary cache miss. + // Failure here only means an unnecessary cache miss. man.writeManifest() catch |err| { log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)}); }; diff --git a/src/link.zig b/src/link.zig index cd41c54cd4..b7b891b5b1 100644 --- a/src/link.zig +++ b/src/link.zig @@ -466,7 +466,11 @@ pub const File = struct { const digest = ch.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| b: { + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| b: { log.debug("archive new_digest={} readFile error: {}", .{ digest, @errorName(err) }); break :b prev_digest_buf[0..0]; }; @@ -512,7 +516,7 @@ pub const File = struct { const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type); if (bad) return error.UnableToWriteArchive; - directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save archive hash digest file: {}", .{@errorName(err)}); }; diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 5c7975743f..492dbfc8eb 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -854,8 +854,12 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { _ = try man.hit(); digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("COFF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| blk: { + log.debug("COFF LLD new_digest={} error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -1180,9 +1184,9 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling file with the digest. If it fails we can continue; it only + // Update the file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 4fcfe78901..9c4029b3cd 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1326,8 +1326,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("ELF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| blk: { + log.debug("ELF LLD new_digest={} error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -1647,9 +1651,9 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling file with the digest. If it fails we can continue; it only + // Update the file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 20fda846f5..9700a0bdb4 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -419,8 +419,12 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("MachO LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| blk: { + log.debug("MachO LLD new_digest={} error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -674,9 +678,9 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling file with the digest. If it fails we can continue; it only + // Update the file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 320f90de19..2ab757461c 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -310,8 +310,12 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { digest = man.final(); var prev_digest_buf: [digest.len]u8 = undefined; - const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { - log.debug("WASM LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); + const prev_digest: []u8 = Cache.readSmallFile( + directory.handle, + id_symlink_basename, + &prev_digest_buf, + ) catch |err| blk: { + log.debug("WASM LLD new_digest={} error: {}", .{ digest, @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; @@ -424,9 +428,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { } if (!self.base.options.disable_lld_caching) { - // Update the dangling file with the digest. If it fails we can continue; it only + // Update the file with the digest. If it fails we can continue; it only // means that the next invocation will have an unnecessary cache miss. - directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { + Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| { std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); }; // Again failure here only means an unnecessary cache miss. From a31b70c4b8d0bed67463b2f54e74198baa93329f Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 10 Oct 2020 00:46:53 +0200 Subject: [PATCH 20/22] std: Add/Fix/Change parts of big.int * Add an optimized squaring routine under the `sqr` name. Algorithms for squaring bigger numbers efficiently will come in a PR later. * Fix a bug where a multiplication was done twice if the threshold for the use of Karatsuba algorithm was crossed. Add a test to make sure this won't happen again. * Streamline `pow` method, take a `Const` parameter. * Minor tweaks to `pow`, avoid bit-reversing the exponent. --- lib/std/math/big/int.zig | 130 +++++++++++++++++++++++++++------- lib/std/math/big/int_test.zig | 44 +++++++++--- 2 files changed, 137 insertions(+), 37 deletions(-) diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig index 54ad2f55d0..25cafda9ac 100644 --- a/lib/std/math/big/int.zig +++ b/lib/std/math/big/int.zig @@ -446,6 +446,26 @@ pub const Mutable = struct { rma.positive = (a.positive == b.positive); } + /// rma = a * a + /// + /// `rma` may not alias with `a`. + /// + /// Asserts the result fits in `rma`. An upper bound on the number of limbs needed by + /// rma is given by `2 * a.limbs.len + 1`. + /// + /// If `allocator` is provided, it will be used for temporary storage to improve + /// multiplication performance. `error.OutOfMemory` is handled with a fallback algorithm. + pub fn sqrNoAlias(rma: *Mutable, a: Const, opt_allocator: ?*Allocator) void { + assert(rma.limbs.ptr != a.limbs.ptr); // illegal aliasing + + mem.set(Limb, rma.limbs, 0); + + llsquare_basecase(rma.limbs, a.limbs); + + rma.normalize(2 * a.limbs.len + 1); + rma.positive = true; + } + /// q = a / b (rem r) /// /// a / b are floored (rounded towards 0). @@ -1827,7 +1847,28 @@ pub const Managed = struct { rma.setMetadata(m.positive, m.len); } - pub fn pow(rma: *Managed, a: Managed, b: u32) !void { + /// r = a * a + pub fn sqr(rma: *Managed, a: Const) !void { + const needed_limbs = 2 * a.limbs.len + 1; + + if (rma.limbs.ptr == a.limbs.ptr) { + var m = try Managed.initCapacity(rma.allocator, needed_limbs); + errdefer m.deinit(); + var m_mut = m.toMutable(); + m_mut.sqrNoAlias(a, rma.allocator); + m.setMetadata(m_mut.positive, m_mut.len); + + rma.deinit(); + rma.swap(&m); + } else { + try rma.ensureCapacity(needed_limbs); + var rma_mut = rma.toMutable(); + rma_mut.sqrNoAlias(a, rma.allocator); + rma.setMetadata(rma_mut.positive, rma_mut.len); + } + } + + pub fn pow(rma: *Managed, a: Const, b: u32) !void { const needed_limbs = calcPowLimbsBufferLen(a.bitCountAbs(), b); const limbs_buffer = try rma.allocator.alloc(Limb, needed_limbs); @@ -1837,7 +1878,7 @@ pub const Managed = struct { var m = try Managed.initCapacity(rma.allocator, needed_limbs); errdefer m.deinit(); var m_mut = m.toMutable(); - try m_mut.pow(a.toConst(), b, limbs_buffer); + try m_mut.pow(a, b, limbs_buffer); m.setMetadata(m_mut.positive, m_mut.len); rma.deinit(); @@ -1845,7 +1886,7 @@ pub const Managed = struct { } else { try rma.ensureCapacity(needed_limbs); var rma_mut = rma.toMutable(); - try rma_mut.pow(a.toConst(), b, limbs_buffer); + try rma_mut.pow(a, b, limbs_buffer); rma.setMetadata(rma_mut.positive, rma_mut.len); } } @@ -1869,11 +1910,14 @@ fn llmulacc(opt_allocator: ?*Allocator, r: []Limb, a: []const Limb, b: []const L assert(r.len >= x.len + y.len + 1); // 48 is a pretty abitrary size chosen based on performance of a factorial program. - if (x.len > 48) { - if (opt_allocator) |allocator| { - llmulacc_karatsuba(allocator, r, x, y) catch |err| switch (err) { - error.OutOfMemory => {}, // handled below - }; + k_mul: { + if (x.len > 48) { + if (opt_allocator) |allocator| { + llmulacc_karatsuba(allocator, r, x, y) catch |err| switch (err) { + error.OutOfMemory => break :k_mul, // handled below + }; + return; + } } } @@ -2203,6 +2247,42 @@ fn llxor(r: []Limb, a: []const Limb, b: []const Limb) void { } } +/// r MUST NOT alias x. +fn llsquare_basecase(r: []Limb, x: []const Limb) void { + @setRuntimeSafety(debug_safety); + + const x_norm = x; + assert(r.len >= 2 * x_norm.len + 1); + + // Compute the square of a N-limb bigint with only (N^2 + N)/2 + // multiplications by exploting the symmetry of the coefficients around the + // diagonal: + // + // a b c * + // a b c = + // ------------------- + // ca cb cc + + // ba bb bc + + // aa ab ac + // + // Note that: + // - Each mixed-product term appears twice for each column, + // - Squares are always in the 2k (0 <= k < N) column + + for (x_norm) |v, i| { + // Accumulate all the x[i]*x[j] (with x!=j) products + llmulDigit(r[2 * i + 1 ..], x_norm[i + 1 ..], v); + } + + // Each product appears twice, multiply by 2 + llshl(r, r[0 .. 2 * x_norm.len], 1); + + for (x_norm) |v, i| { + // Compute and add the squares + llmulDigit(r[2 * i ..], x[i .. i + 1], v); + } +} + /// Knuth 4.6.3 fn llpow(r: []Limb, a: []const Limb, b: u32, tmp_limbs: []Limb) void { var tmp1: []Limb = undefined; @@ -2212,9 +2292,9 @@ fn llpow(r: []Limb, a: []const Limb, b: u32, tmp_limbs: []Limb) void { // variable, use the output limbs and another temporary set to overcome this // limitation. // The initial assignment makes the result end in `r` so an extra memory - // copy is saved, each 1 flips the index twice so it's a no-op so count the - // 0. - const b_leading_zeros = @intCast(u5, @clz(u32, b)); + // copy is saved, each 1 flips the index twice so it's only the zeros that + // matter. + const b_leading_zeros = @clz(u32, b); const exp_zeros = @popCount(u32, ~b) - b_leading_zeros; if (exp_zeros & 1 != 0) { tmp1 = tmp_limbs; @@ -2224,32 +2304,28 @@ fn llpow(r: []Limb, a: []const Limb, b: u32, tmp_limbs: []Limb) void { tmp2 = tmp_limbs; } - const a_norm = a[0..llnormalize(a)]; - - mem.copy(Limb, tmp1, a_norm); - mem.set(Limb, tmp1[a_norm.len..], 0); + mem.copy(Limb, tmp1, a); + mem.set(Limb, tmp1[a.len..], 0); // Scan the exponent as a binary number, from left to right, dropping the // most significant bit set. - const exp_bits = @intCast(u5, 31 - b_leading_zeros); - var exp = @bitReverse(u32, b) >> 1 + b_leading_zeros; + // Square the result if the current bit is zero, square and multiply by a if + // it is one. + var exp_bits = 32 - 1 - b_leading_zeros; + var exp = b << @intCast(u5, 1 + b_leading_zeros); - var i: u5 = 0; + var i: usize = 0; while (i < exp_bits) : (i += 1) { // Square - { - mem.set(Limb, tmp2, 0); - const op = tmp1[0..llnormalize(tmp1)]; - llmulacc(null, tmp2, op, op); - mem.swap([]Limb, &tmp1, &tmp2); - } + mem.set(Limb, tmp2, 0); + llsquare_basecase(tmp2, tmp1[0..llnormalize(tmp1)]); + mem.swap([]Limb, &tmp1, &tmp2); // Multiply by a - if (exp & 1 != 0) { + if (@shlWithOverflow(u32, exp, 1, &exp)) { mem.set(Limb, tmp2, 0); - llmulacc(null, tmp2, tmp1[0..llnormalize(tmp1)], a_norm); + llmulacc(null, tmp2, tmp1[0..llnormalize(tmp1)], a); mem.swap([]Limb, &tmp1, &tmp2); } - exp >>= 1; } } diff --git a/lib/std/math/big/int_test.zig b/lib/std/math/big/int_test.zig index 5d07bee9b5..1f4bd65974 100644 --- a/lib/std/math/big/int_test.zig +++ b/lib/std/math/big/int_test.zig @@ -720,6 +720,27 @@ test "big.int mul 0*0" { testing.expect((try c.to(u32)) == 0); } +test "big.int mul large" { + var a = try Managed.initCapacity(testing.allocator, 50); + defer a.deinit(); + var b = try Managed.initCapacity(testing.allocator, 100); + defer b.deinit(); + var c = try Managed.initCapacity(testing.allocator, 100); + defer c.deinit(); + + // Generate a number that's large enough to cross the thresholds for the use + // of subquadratic algorithms + for (a.limbs) |*p| { + p.* = std.math.maxInt(Limb); + } + a.setMetadata(true, 50); + + try b.mul(a.toConst(), a.toConst()); + try c.sqr(a.toConst()); + + testing.expect(b.eq(c)); +} + test "big.int div single-single no rem" { var a = try Managed.initSet(testing.allocator, 50); defer a.deinit(); @@ -1483,11 +1504,14 @@ test "big.int const to managed" { test "big.int pow" { { - var a = try Managed.initSet(testing.allocator, 10); + var a = try Managed.initSet(testing.allocator, -3); defer a.deinit(); - try a.pow(a, 8); - testing.expectEqual(@as(u32, 100000000), try a.to(u32)); + try a.pow(a.toConst(), 3); + testing.expectEqual(@as(i32, -27), try a.to(i32)); + + try a.pow(a.toConst(), 4); + testing.expectEqual(@as(i32, 531441), try a.to(i32)); } { var a = try Managed.initSet(testing.allocator, 10); @@ -1497,9 +1521,9 @@ test "big.int pow" { defer y.deinit(); // y and a are not aliased - try y.pow(a, 123); + try y.pow(a.toConst(), 123); // y and a are aliased - try a.pow(a, 123); + try a.pow(a.toConst(), 123); testing.expect(a.eq(y)); @@ -1517,18 +1541,18 @@ test "big.int pow" { var a = try Managed.initSet(testing.allocator, 0); defer a.deinit(); - try a.pow(a, 100); + try a.pow(a.toConst(), 100); testing.expectEqual(@as(i32, 0), try a.to(i32)); try a.set(1); - try a.pow(a, 0); + try a.pow(a.toConst(), 0); testing.expectEqual(@as(i32, 1), try a.to(i32)); - try a.pow(a, 100); + try a.pow(a.toConst(), 100); testing.expectEqual(@as(i32, 1), try a.to(i32)); try a.set(-1); - try a.pow(a, 15); + try a.pow(a.toConst(), 15); testing.expectEqual(@as(i32, -1), try a.to(i32)); - try a.pow(a, 16); + try a.pow(a.toConst(), 16); testing.expectEqual(@as(i32, 1), try a.to(i32)); } } From 53c63bdb73d9fbc5a54afb4977bb975b03c4c9cc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 10 Oct 2020 10:52:36 +0200 Subject: [PATCH 21/22] Update WASI preopens doc section to use GPA Signed-off-by: Jakub Konka --- doc/langref.html.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 021fc76289..7caae4afff 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -9905,7 +9905,10 @@ const std = @import("std"); const PreopenList = std.fs.wasi.PreopenList; pub fn main() !void { - var preopens = PreopenList.init(std.heap.page_allocator); + var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; + const gpa = &general_purpose_allocator.allocator; + + var preopens = PreopenList.init(gpa); defer preopens.deinit(); try preopens.populate(); From 2ab0c7391a871a3063f825e08e02ea2a8e9269e9 Mon Sep 17 00:00:00 2001 From: Vignesh Rajagopalan Date: Mon, 12 Oct 2020 14:29:43 +0530 Subject: [PATCH 22/22] Rename .macosx to .macos --- lib/std/c.zig | 8 +++---- lib/std/debug.zig | 4 ++-- lib/std/dynamic_library.zig | 4 ++-- lib/std/event/loop.zig | 24 ++++++++++----------- lib/std/fs.zig | 10 ++++----- lib/std/fs/get_app_data_dir.zig | 2 +- lib/std/fs/test.zig | 2 +- lib/std/fs/watch.zig | 8 +++---- lib/std/os.zig | 18 ++++++++-------- lib/std/os/bits.zig | 2 +- lib/std/os/test.zig | 2 +- lib/std/packed_int_array.zig | 4 ++-- lib/std/process.zig | 6 +++--- lib/std/special/compiler_rt/clear_cache.zig | 2 +- lib/std/target.zig | 16 +++++++------- lib/std/time.zig | 2 +- lib/std/zig/cross_target.zig | 6 +++--- lib/std/zig/system.zig | 2 +- src/codegen/llvm.zig | 2 +- src/link/MachO.zig | 2 +- src/target.zig | 6 +++--- src/type.zig | 2 +- src/zig_llvm.cpp | 4 +++- test/stack_traces.zig | 2 +- test/stage2/test.zig | 2 +- test/tests.zig | 2 +- 26 files changed, 73 insertions(+), 71 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index a75fcaa84b..1fd3c593cc 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -22,7 +22,7 @@ pub usingnamespace @import("os/bits.zig"); pub usingnamespace switch (std.Target.current.os.tag) { .linux => @import("c/linux.zig"), .windows => @import("c/windows.zig"), - .macosx, .ios, .tvos, .watchos => @import("c/darwin.zig"), + .macos, .ios, .tvos, .watchos => @import("c/darwin.zig"), .freebsd, .kfreebsd => @import("c/freebsd.zig"), .netbsd => @import("c/netbsd.zig"), .dragonfly => @import("c/dragonfly.zig"), @@ -122,7 +122,7 @@ pub extern "c" fn readlink(noalias path: [*:0]const u8, noalias buf: [*]u8, bufs pub extern "c" fn readlinkat(dirfd: fd_t, noalias path: [*:0]const u8, noalias buf: [*]u8, bufsize: usize) isize; pub usingnamespace switch (builtin.os.tag) { - .macosx, .ios, .watchos, .tvos => struct { + .macos, .ios, .watchos, .tvos => struct { pub const realpath = @"realpath$DARWIN_EXTSN"; pub const fstatat = @"fstatat$INODE64"; }, @@ -189,7 +189,7 @@ pub usingnamespace switch (builtin.os.tag) { pub const sigprocmask = __sigprocmask14; pub const stat = __stat50; }, - .macosx, .ios, .watchos, .tvos => struct { + .macos, .ios, .watchos, .tvos => struct { // XXX: close -> close$NOCANCEL // XXX: getdirentries -> _getdirentries64 pub extern "c" fn clock_getres(clk_id: c_int, tp: *timespec) c_int; @@ -252,7 +252,7 @@ pub usingnamespace switch (builtin.os.tag) { .linux, .freebsd, .kfreebsd, .netbsd, .openbsd => struct { pub extern "c" fn malloc_usable_size(?*const c_void) usize; }, - .macosx, .ios, .watchos, .tvos => struct { + .macos, .ios, .watchos, .tvos => struct { pub extern "c" fn malloc_size(?*const c_void) usize; }, else => struct {}, diff --git a/lib/std/debug.zig b/lib/std/debug.zig index be57c0b2fd..8dc2938894 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -654,7 +654,7 @@ pub fn openSelfDebugInfo(allocator: *mem.Allocator) anyerror!DebugInfo { .freebsd, .netbsd, .dragonfly, - .macosx, + .macos, .windows, => return DebugInfo.init(allocator), else => @compileError("openSelfDebugInfo unsupported for this platform"), @@ -1320,7 +1320,7 @@ const SymbolInfo = struct { }; pub const ModuleDebugInfo = switch (builtin.os.tag) { - .macosx, .ios, .watchos, .tvos => struct { + .macos, .ios, .watchos, .tvos => struct { base_address: usize, mapped_memory: []const u8, symbols: []const MachoSymbol, diff --git a/lib/std/dynamic_library.zig b/lib/std/dynamic_library.zig index 238854f07f..07be46a8e1 100644 --- a/lib/std/dynamic_library.zig +++ b/lib/std/dynamic_library.zig @@ -19,7 +19,7 @@ const max = std.math.max; pub const DynLib = switch (builtin.os.tag) { .linux => if (builtin.link_libc) DlDynlib else ElfDynLib, .windows => WindowsDynLib, - .macosx, .tvos, .watchos, .ios, .freebsd => DlDynlib, + .macos, .tvos, .watchos, .ios, .freebsd => DlDynlib, else => void, }; @@ -404,7 +404,7 @@ test "dynamic_library" { const libname = switch (builtin.os.tag) { .linux, .freebsd => "invalid_so.so", .windows => "invalid_dll.dll", - .macosx, .tvos, .watchos, .ios => "invalid_dylib.dylib", + .macos, .tvos, .watchos, .ios => "invalid_dylib.dylib", else => return error.SkipZigTest, }; diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index a064f711e2..5b10335535 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -66,7 +66,7 @@ pub const Loop = struct { }; pub const EventFd = switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => KEventFd, + .macos, .freebsd, .netbsd, .dragonfly => KEventFd, .linux => struct { base: ResumeNode, epoll_op: u32, @@ -85,7 +85,7 @@ pub const Loop = struct { }; pub const Basic = switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => KEventBasic, + .macos, .freebsd, .netbsd, .dragonfly => KEventBasic, .linux => struct { base: ResumeNode, }, @@ -259,7 +259,7 @@ pub const Loop = struct { self.extra_threads[extra_thread_index] = try Thread.spawn(self, workerRun); } }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { self.os_data.kqfd = try os.kqueue(); errdefer os.close(self.os_data.kqfd); @@ -384,7 +384,7 @@ pub const Loop = struct { while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); os.close(self.os_data.epollfd); }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { os.close(self.os_data.kqfd); }, .windows => { @@ -478,7 +478,7 @@ pub const Loop = struct { .linux => { self.linuxWaitFd(fd, os.EPOLLET | os.EPOLLONESHOT | os.EPOLLIN); }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { self.bsdWaitKev(@intCast(usize, fd), os.EVFILT_READ, os.EV_ONESHOT); }, else => @compileError("Unsupported OS"), @@ -490,7 +490,7 @@ pub const Loop = struct { .linux => { self.linuxWaitFd(fd, os.EPOLLET | os.EPOLLONESHOT | os.EPOLLOUT); }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { self.bsdWaitKev(@intCast(usize, fd), os.EVFILT_WRITE, os.EV_ONESHOT); }, else => @compileError("Unsupported OS"), @@ -502,7 +502,7 @@ pub const Loop = struct { .linux => { self.linuxWaitFd(fd, os.EPOLLET | os.EPOLLONESHOT | os.EPOLLOUT | os.EPOLLIN); }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { self.bsdWaitKev(@intCast(usize, fd), os.EVFILT_READ, os.EV_ONESHOT); self.bsdWaitKev(@intCast(usize, fd), os.EVFILT_WRITE, os.EV_ONESHOT); }, @@ -571,7 +571,7 @@ pub const Loop = struct { const eventfd_node = &resume_stack_node.data; eventfd_node.base.handle = next_tick_node.data; switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { const kevent_array = @as(*const [1]os.Kevent, &eventfd_node.kevent); const empty_kevs = &[0]os.Kevent{}; _ = os.kevent(self.os_data.kqfd, kevent_array, empty_kevs, null) catch { @@ -633,7 +633,7 @@ pub const Loop = struct { if (!builtin.single_threaded) { switch (builtin.os.tag) { .linux, - .macosx, + .macos, .freebsd, .netbsd, .dragonfly, @@ -725,7 +725,7 @@ pub const Loop = struct { } return; }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { const final_kevent = @as(*const [1]os.Kevent, &self.os_data.final_kevent); const empty_kevs = &[0]os.Kevent{}; // cannot fail because we already added it and this just enables it @@ -1218,7 +1218,7 @@ pub const Loop = struct { } } }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { var eventlist: [1]os.Kevent = undefined; const empty_kevs = &[0]os.Kevent{}; const count = os.kevent(self.os_data.kqfd, empty_kevs, eventlist[0..], null) catch unreachable; @@ -1344,7 +1344,7 @@ pub const Loop = struct { const OsData = switch (builtin.os.tag) { .linux => LinuxOsData, - .macosx, .freebsd, .netbsd, .dragonfly => KEventData, + .macos, .freebsd, .netbsd, .dragonfly => KEventData, .windows => struct { io_port: windows.HANDLE, extra_thread_count: usize, diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 0f8c6e0d24..6fd188e1f9 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -39,7 +39,7 @@ pub const Watch = @import("fs/watch.zig").Watch; /// fit into a UTF-8 encoded array of this length. /// The byte count includes room for a null sentinel byte. pub const MAX_PATH_BYTES = switch (builtin.os.tag) { - .linux, .macosx, .ios, .freebsd, .netbsd, .dragonfly => os.PATH_MAX, + .linux, .macos, .ios, .freebsd, .netbsd, .dragonfly => os.PATH_MAX, // Each UTF-16LE character may be expanded to 3 UTF-8 bytes. // If it would require 4 UTF-8 bytes, then there would be a surrogate // pair in the UTF-16LE, and we (over)account 3 bytes for it that way. @@ -303,7 +303,7 @@ pub const Dir = struct { const IteratorError = error{AccessDenied} || os.UnexpectedError; pub const Iterator = switch (builtin.os.tag) { - .macosx, .ios, .freebsd, .netbsd, .dragonfly => struct { + .macos, .ios, .freebsd, .netbsd, .dragonfly => struct { dir: Dir, seek: i64, buf: [8192]u8, // TODO align(@alignOf(os.dirent)), @@ -318,7 +318,7 @@ pub const Dir = struct { /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. pub fn next(self: *Self) Error!?Entry { switch (builtin.os.tag) { - .macosx, .ios => return self.nextDarwin(), + .macos, .ios => return self.nextDarwin(), .freebsd, .netbsd, .dragonfly => return self.nextBsd(), else => @compileError("unimplemented"), } @@ -615,7 +615,7 @@ pub const Dir = struct { pub fn iterate(self: Dir) Iterator { switch (builtin.os.tag) { - .macosx, .ios, .freebsd, .netbsd, .dragonfly => return Iterator{ + .macos, .ios, .freebsd, .netbsd, .dragonfly => return Iterator{ .dir = self, .seek = 0, .index = 0, @@ -1302,7 +1302,7 @@ pub const Dir = struct { error.AccessDenied => |e| switch (builtin.os.tag) { // non-Linux POSIX systems return EPERM when trying to delete a directory, so // we need to handle that case specifically and translate the error - .macosx, .ios, .freebsd, .netbsd, .dragonfly => { + .macos, .ios, .freebsd, .netbsd, .dragonfly => { // Don't follow symlinks to match unlinkat (which acts on symlinks rather than follows them) const fstat = os.fstatatZ(self.fd, sub_path_c, os.AT_SYMLINK_NOFOLLOW) catch return e; const is_dir = fstat.mode & os.S_IFMT == os.S_IFDIR; diff --git a/lib/std/fs/get_app_data_dir.zig b/lib/std/fs/get_app_data_dir.zig index 17d365f684..6f8dae07c5 100644 --- a/lib/std/fs/get_app_data_dir.zig +++ b/lib/std/fs/get_app_data_dir.zig @@ -42,7 +42,7 @@ pub fn getAppDataDir(allocator: *mem.Allocator, appname: []const u8) GetAppDataD else => return error.AppDataDirUnavailable, } }, - .macosx => { + .macos => { const home_dir = os.getenv("HOME") orelse { // TODO look in /etc/passwd return error.AppDataDirUnavailable; diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index 60cd380444..860da8ef16 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -143,7 +143,7 @@ fn contains(entries: *const std.ArrayList(Dir.Entry), el: Dir.Entry) bool { test "Dir.realpath smoke test" { switch (builtin.os.tag) { - .linux, .windows, .macosx, .ios, .watchos, .tvos => {}, + .linux, .windows, .macos, .ios, .watchos, .tvos => {}, else => return error.SkipZigTest, } diff --git a/lib/std/fs/watch.zig b/lib/std/fs/watch.zig index 269cdec6d8..8b709c2d4b 100644 --- a/lib/std/fs/watch.zig +++ b/lib/std/fs/watch.zig @@ -49,7 +49,7 @@ pub fn Watch(comptime V: type) type { const OsData = switch (builtin.os.tag) { // TODO https://github.com/ziglang/zig/issues/3778 - .macosx, .freebsd, .netbsd, .dragonfly => KqOsData, + .macos, .freebsd, .netbsd, .dragonfly => KqOsData, .linux => LinuxOsData, .windows => WindowsOsData, @@ -160,7 +160,7 @@ pub fn Watch(comptime V: type) type { return self; }, - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { self.* = Self{ .allocator = allocator, .channel = channel, @@ -178,7 +178,7 @@ pub fn Watch(comptime V: type) type { /// All addFile calls and removeFile calls must have completed. pub fn deinit(self: *Self) void { switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => { + .macos, .freebsd, .netbsd, .dragonfly => { // TODO we need to cancel the frames before destroying the lock self.os_data.table_lock.deinit(); var it = self.os_data.file_table.iterator(); @@ -229,7 +229,7 @@ pub fn Watch(comptime V: type) type { pub fn addFile(self: *Self, file_path: []const u8, value: V) !?V { switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => return addFileKEvent(self, file_path, value), + .macos, .freebsd, .netbsd, .dragonfly => return addFileKEvent(self, file_path, value), .linux => return addFileLinux(self, file_path, value), .windows => return addFileWindows(self, file_path, value), else => @compileError("Unsupported OS"), diff --git a/lib/std/os.zig b/lib/std/os.zig index 9f6c012090..bc7dae031c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -62,7 +62,7 @@ pub const system = if (@hasDecl(root, "os") and root.os != @This()) else if (builtin.link_libc) std.c else switch (builtin.os.tag) { - .macosx, .ios, .watchos, .tvos => darwin, + .macos, .ios, .watchos, .tvos => darwin, .freebsd => freebsd, .linux => linux, .netbsd => netbsd, @@ -354,7 +354,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { // Prevents EINVAL. const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, - .macosx, .ios, .watchos, .tvos => math.maxInt(i32), + .macos, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, buf.len); @@ -582,7 +582,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { /// On these systems, the read races with concurrent writes to the same file descriptor. pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize { const have_pread_but_not_preadv = switch (std.Target.current.os.tag) { - .windows, .macosx, .ios, .watchos, .tvos => true, + .windows, .macos, .ios, .watchos, .tvos => true, else => false, }; if (have_pread_but_not_preadv) { @@ -709,7 +709,7 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, - .macosx, .ios, .watchos, .tvos => math.maxInt(i32), + .macos, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, bytes.len); @@ -863,7 +863,7 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { // Prevent EINVAL. const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, - .macosx, .ios, .watchos, .tvos => math.maxInt(i32), + .macos, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(isize), }; const adjusted_len = math.min(max_count, bytes.len); @@ -915,7 +915,7 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { /// If `iov.len` is larger than will fit in a `u31`, a partial write will occur. pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) PWriteError!usize { const have_pwrite_but_not_pwritev = switch (std.Target.current.os.tag) { - .windows, .macosx, .ios, .watchos, .tvos => true, + .windows, .macos, .ios, .watchos, .tvos => true, else => false, }; @@ -4091,7 +4091,7 @@ pub fn getFdPath(fd: fd_t, out_buffer: *[MAX_PATH_BYTES]u8) RealPathError![]u8 { const end_index = std.unicode.utf16leToUtf8(out_buffer, wide_slice) catch unreachable; return out_buffer[0..end_index]; }, - .macosx, .ios, .watchos, .tvos => { + .macos, .ios, .watchos, .tvos => { // On macOS, we can use F_GETPATH fcntl command to query the OS for // the path to the file descriptor. @memset(out_buffer, 0, MAX_PATH_BYTES); @@ -4688,7 +4688,7 @@ pub fn sendfile( }); const max_count = switch (std.Target.current.os.tag) { .linux => 0x7ffff000, - .macosx, .ios, .watchos, .tvos => math.maxInt(i32), + .macos, .ios, .watchos, .tvos => math.maxInt(i32), else => math.maxInt(size_t), }; @@ -4846,7 +4846,7 @@ pub fn sendfile( } } }, - .macosx, .ios, .tvos, .watchos => sf: { + .macos, .ios, .tvos, .watchos => sf: { var hdtr_data: std.c.sf_hdtr = undefined; var hdtr: ?*std.c.sf_hdtr = null; if (headers.len != 0 or trailers.len != 0) { diff --git a/lib/std/os/bits.zig b/lib/std/os/bits.zig index 177b7daad2..3647f67f2a 100644 --- a/lib/std/os/bits.zig +++ b/lib/std/os/bits.zig @@ -12,7 +12,7 @@ const std = @import("std"); const root = @import("root"); pub usingnamespace switch (std.Target.current.os.tag) { - .macosx, .ios, .tvos, .watchos => @import("bits/darwin.zig"), + .macos, .ios, .tvos, .watchos => @import("bits/darwin.zig"), .dragonfly => @import("bits/dragonfly.zig"), .freebsd => @import("bits/freebsd.zig"), .linux => @import("bits/linux.zig"), diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index aee1b9549a..58c2b311b1 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -360,7 +360,7 @@ fn iter_fn(info: *dl_phdr_info, size: usize, counter: *usize) IterFnError!void { } test "dl_iterate_phdr" { - if (builtin.os.tag == .windows or builtin.os.tag == .wasi or builtin.os.tag == .macosx) + if (builtin.os.tag == .windows or builtin.os.tag == .wasi or builtin.os.tag == .macos) return error.SkipZigTest; var counter: usize = 0; diff --git a/lib/std/packed_int_array.zig b/lib/std/packed_int_array.zig index f25ff0b1b8..2f2b671692 100644 --- a/lib/std/packed_int_array.zig +++ b/lib/std/packed_int_array.zig @@ -618,7 +618,7 @@ test "PackedIntArray at end of available memory" { if (we_are_testing_this_with_stage1_which_leaks_comptime_memory) return error.SkipZigTest; switch (builtin.os.tag) { - .linux, .macosx, .ios, .freebsd, .netbsd, .windows => {}, + .linux, .macos, .ios, .freebsd, .netbsd, .windows => {}, else => return, } const PackedArray = PackedIntArray(u3, 8); @@ -639,7 +639,7 @@ test "PackedIntSlice at end of available memory" { if (we_are_testing_this_with_stage1_which_leaks_comptime_memory) return error.SkipZigTest; switch (builtin.os.tag) { - .linux, .macosx, .ios, .freebsd, .netbsd, .windows => {}, + .linux, .macos, .ios, .freebsd, .netbsd, .windows => {}, else => return, } const PackedSlice = PackedIntSlice(u11); diff --git a/lib/std/process.zig b/lib/std/process.zig index 2813d8cbab..3a499f3e24 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -585,7 +585,7 @@ pub const UserInfo = struct { /// POSIX function which gets a uid from username. pub fn getUserInfo(name: []const u8) !UserInfo { return switch (builtin.os.tag) { - .linux, .macosx, .watchos, .tvos, .ios, .freebsd, .netbsd => posixGetUserInfo(name), + .linux, .macos, .watchos, .tvos, .ios, .freebsd, .netbsd => posixGetUserInfo(name), else => @compileError("Unsupported OS"), }; } @@ -688,7 +688,7 @@ pub fn getBaseAddress() usize { const phdr = os.system.getauxval(std.elf.AT_PHDR); return phdr - @sizeOf(std.elf.Ehdr); }, - .macosx, .freebsd, .netbsd => { + .macos, .freebsd, .netbsd => { return @ptrToInt(&std.c._mh_execute_header); }, .windows => return @ptrToInt(os.windows.kernel32.GetModuleHandleW(null)), @@ -733,7 +733,7 @@ pub fn getSelfExeSharedLibPaths(allocator: *Allocator) error{OutOfMemory}![][:0] }.callback); return paths.toOwnedSlice(); }, - .macosx, .ios, .watchos, .tvos => { + .macos, .ios, .watchos, .tvos => { var paths = List.init(allocator); errdefer { const slice = paths.toOwnedSlice(); diff --git a/lib/std/special/compiler_rt/clear_cache.zig b/lib/std/special/compiler_rt/clear_cache.zig index d862f739d3..4b00721868 100644 --- a/lib/std/special/compiler_rt/clear_cache.zig +++ b/lib/std/special/compiler_rt/clear_cache.zig @@ -44,7 +44,7 @@ pub fn clear_cache(start: usize, end: usize) callconv(.C) void { else => false, }; const apple = switch (os) { - .ios, .macosx, .watchos, .tvos => true, + .ios, .macos, .watchos, .tvos => true, else => false, }; if (x86) { diff --git a/lib/std/target.zig b/lib/std/target.zig index 99617c6d0e..4b9a6b4458 100644 --- a/lib/std/target.zig +++ b/lib/std/target.zig @@ -31,7 +31,7 @@ pub const Target = struct { kfreebsd, linux, lv2, - macosx, + macos, netbsd, openbsd, solaris, @@ -61,7 +61,7 @@ pub const Target = struct { pub fn isDarwin(tag: Tag) bool { return switch (tag) { - .ios, .macosx, .watchos, .tvos => true, + .ios, .macos, .watchos, .tvos => true, else => false, }; } @@ -234,7 +234,7 @@ pub const Target = struct { .max = .{ .major = 12, .minor = 1 }, }, }, - .macosx => return .{ + .macos => return .{ .semver = .{ .min = .{ .major = 10, .minor = 13 }, .max = .{ .major = 10, .minor = 15, .patch = 3 }, @@ -312,7 +312,7 @@ pub const Target = struct { .windows => return TaggedVersionRange{ .windows = self.version_range.windows }, .freebsd, - .macosx, + .macos, .ios, .tvos, .watchos, @@ -341,7 +341,7 @@ pub const Target = struct { return switch (os.tag) { .freebsd, .netbsd, - .macosx, + .macos, .ios, .tvos, .watchos, @@ -450,7 +450,7 @@ pub const Target = struct { .other, => return .eabi, .openbsd, - .macosx, + .macos, .freebsd, .ios, .tvos, @@ -1277,7 +1277,7 @@ pub const Target = struct { .ios, .tvos, .watchos, - .macosx, + .macos, .uefi, .windows, .emscripten, @@ -1450,7 +1450,7 @@ pub const Target = struct { .ios, .tvos, .watchos, - .macosx, + .macos, .uefi, .windows, .emscripten, diff --git a/lib/std/time.zig b/lib/std/time.zig index ae128f6b0c..344f192784 100644 --- a/lib/std/time.zig +++ b/lib/std/time.zig @@ -143,7 +143,7 @@ pub const Timer = struct { /// be less precise frequency: switch (builtin.os.tag) { .windows => u64, - .macosx, .ios, .tvos, .watchos => os.darwin.mach_timebase_info_data, + .macos, .ios, .tvos, .watchos => os.darwin.mach_timebase_info_data, else => void, }, resolution: u64, diff --git a/lib/std/zig/cross_target.zig b/lib/std/zig/cross_target.zig index f1ae3457a5..3d39c8338b 100644 --- a/lib/std/zig/cross_target.zig +++ b/lib/std/zig/cross_target.zig @@ -137,7 +137,7 @@ pub const CrossTarget = struct { }, .freebsd, - .macosx, + .macos, .ios, .tvos, .watchos, @@ -578,7 +578,7 @@ pub const CrossTarget = struct { const os = switch (self.getOsTag()) { .windows => "windows", .linux => "linux", - .macosx => "macos", + .macos => "macos", else => return error.UnsupportedVcpkgOperatingSystem, }; @@ -718,7 +718,7 @@ pub const CrossTarget = struct { => return error.InvalidOperatingSystemVersion, .freebsd, - .macosx, + .macos, .ios, .tvos, .watchos, diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index b1947058cc..33ef137eb6 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -267,7 +267,7 @@ pub const NativeTargetInfo = struct { os.version_range.windows.max = @intToEnum(Target.Os.WindowsVersion, version); os.version_range.windows.min = @intToEnum(Target.Os.WindowsVersion, version); }, - .macosx => { + .macos => { var scbuf: [32]u8 = undefined; var size: usize = undefined; diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 01fa0baf02..f87a7a940c 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -69,7 +69,7 @@ pub fn targetTriple(allocator: *Allocator, target: std.Target) ![]u8 { .kfreebsd => "kfreebsd", .linux => "linux", .lv2 => "lv2", - .macosx => "macosx", + .macos => "macosx", .netbsd => "netbsd", .openbsd => "openbsd", .solaris => "solaris", diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 9700a0bdb4..0d3de89d58 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -525,7 +525,7 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { try argv.append(darwinArchString(target.cpu.arch)); switch (target.os.tag) { - .macosx => { + .macos => { try argv.append("-macosx_version_min"); }, .ios, .tvos, .watchos => switch (target.cpu.arch) { diff --git a/src/target.zig b/src/target.zig index d009cf8556..ce3a200d2b 100644 --- a/src/target.zig +++ b/src/target.zig @@ -123,14 +123,14 @@ pub fn cannotDynamicLink(target: std.Target) bool { /// since this is the stable syscall interface. pub fn osRequiresLibC(target: std.Target) bool { return switch (target.os.tag) { - .freebsd, .netbsd, .dragonfly, .macosx, .ios, .watchos, .tvos => true, + .freebsd, .netbsd, .dragonfly, .macos, .ios, .watchos, .tvos => true, else => false, }; } pub fn libcNeedsLibUnwind(target: std.Target) bool { return switch (target.os.tag) { - .macosx, + .macos, .ios, .watchos, .tvos, @@ -197,7 +197,7 @@ pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType { .kfreebsd => .KFreeBSD, .linux => .Linux, .lv2 => .Lv2, - .macosx => .MacOSX, + .macos => .MacOSX, .netbsd => .NetBSD, .openbsd => .OpenBSD, .solaris => .Solaris, diff --git a/src/type.zig b/src/type.zig index c22edaa561..f07df290fc 100644 --- a/src/type.zig +++ b/src/type.zig @@ -3104,7 +3104,7 @@ pub const CType = enum { }, .linux, - .macosx, + .macos, .freebsd, .netbsd, .dragonfly, diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp index 78082d16ba..decd11e0ab 100644 --- a/src/zig_llvm.cpp +++ b/src/zig_llvm.cpp @@ -817,7 +817,9 @@ const char *ZigLLVMGetVendorTypeName(ZigLLVM_VendorType vendor) { } const char *ZigLLVMGetOSTypeName(ZigLLVM_OSType os) { - return (const char*)Triple::getOSTypeName((Triple::OSType)os).bytes_begin(); + const char* name = (const char*)Triple::getOSTypeName((Triple::OSType)os).bytes_begin(); + if (strcmp(name, "macosx") == 0) return "macos"; + return name; } const char *ZigLLVMGetEnvironmentTypeName(ZigLLVM_EnvironmentType env_type) { diff --git a/test/stack_traces.zig b/test/stack_traces.zig index 496be05138..640cc0904c 100644 --- a/test/stack_traces.zig +++ b/test/stack_traces.zig @@ -308,7 +308,7 @@ pub fn addCases(cases: *tests.StackTracesContext) void { }, ); }, - .macosx => { + .macos => { cases.addCase( "return", source_return, diff --git a/test/stage2/test.zig b/test/stage2/test.zig index 2e3b570570..5f5fe76d17 100644 --- a/test/stage2/test.zig +++ b/test/stage2/test.zig @@ -13,7 +13,7 @@ const linux_x64 = std.zig.CrossTarget{ const macosx_x64 = std.zig.CrossTarget{ .cpu_arch = .x86_64, - .os_tag = .macosx, + .os_tag = .macos, }; const linux_riscv64 = std.zig.CrossTarget{ diff --git a/test/tests.zig b/test/tests.zig index 58b2b50094..09001b02a1 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -234,7 +234,7 @@ const test_targets = blk: { TestTarget{ .target = .{ .cpu_arch = .x86_64, - .os_tag = .macosx, + .os_tag = .macos, .abi = .gnu, }, // https://github.com/ziglang/zig/issues/3295