diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index e2b8683609..2cc3463e66 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -236,6 +236,8 @@ pub fn main() !void { graph.debug_compiler_runtime_libs = true; } else if (mem.eql(u8, arg, "--debug-compile-errors")) { builder.debug_compile_errors = true; + } else if (mem.eql(u8, arg, "--debug-incremental")) { + builder.debug_incremental = true; } else if (mem.eql(u8, arg, "--system")) { // The usage text shows another argument after this parameter // but it is handled by the parent process. The build runner diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 55d13dfd85..3bfd8c9a7d 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -59,6 +59,7 @@ pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, args: ?[]const []const u8 = null, debug_log_scopes: []const []const u8 = &.{}, debug_compile_errors: bool = false, +debug_incremental: bool = false, debug_pkg_config: bool = false, /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes, /// in particular at `Step` creation. @@ -385,6 +386,7 @@ fn createChildOnly( .cache_root = parent.cache_root, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, + .debug_incremental = parent.debug_incremental, .debug_pkg_config = parent.debug_pkg_config, .enable_darling = parent.enable_darling, .enable_qemu = parent.enable_qemu, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 066167905d..7e8b782932 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1447,6 +1447,10 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { try zig_args.append("--debug-compile-errors"); } + if (b.debug_incremental) { + try zig_args.append("--debug-incremental"); + } + if (b.verbose_cimport) try zig_args.append("--verbose-cimport"); if (b.verbose_air) try zig_args.append("--verbose-air"); if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); diff --git a/src/Compilation.zig b/src/Compilation.zig index 6a546648b2..8f986a5cdf 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -190,6 +190,8 @@ time_report: bool, stack_report: bool, debug_compiler_runtime_libs: bool, debug_compile_errors: bool, +/// Do not check this field directly. Instead, use the `debugIncremental` wrapper function. +debug_incremental: bool, incremental: bool, alloc_failure_occurred: bool = false, last_update_was_cache_hit: bool = false, @@ -768,6 +770,14 @@ pub const Directories = struct { } }; +/// This small wrapper function just checks whether debug extensions are enabled before checking +/// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller, +/// preventing debugging features from making it into release builds of the compiler. +pub inline fn debugIncremental(comp: *const Compilation) bool { + if (!build_options.enable_debug_extensions) return false; + return comp.debug_incremental; +} + pub const default_stack_protector_buffer_size = target_util.default_stack_protector_buffer_size; pub const SemaError = Zcu.SemaError; @@ -1598,6 +1608,7 @@ pub const CreateOptions = struct { verbose_llvm_cpu_features: bool = false, debug_compiler_runtime_libs: bool = false, debug_compile_errors: bool = false, + debug_incremental: bool = false, incremental: bool = false, /// Normally when you create a `Compilation`, Zig will automatically build /// and link in required dependencies, such as compiler-rt and libc. When @@ -1968,6 +1979,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .test_name_prefix = options.test_name_prefix, .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs, .debug_compile_errors = options.debug_compile_errors, + .debug_incremental = options.debug_incremental, .incremental = options.incremental, .root_name = root_name, .sysroot = sysroot, diff --git a/src/IncrementalDebugServer.zig b/src/IncrementalDebugServer.zig new file mode 100644 index 0000000000..69ebc3752e --- /dev/null +++ b/src/IncrementalDebugServer.zig @@ -0,0 +1,383 @@ +//! This is a simple TCP server which exposes a REPL useful for debugging incremental compilation +//! issues. Eventually, this logic should move into `std.zig.Client`/`std.zig.Server` or something +//! similar, but for now, this works. The server is enabled by the '--debug-incremental' CLI flag. +//! The easiest way to interact with the REPL is to use `telnet`: +//! ``` +//! telnet "::1" 7623 +//! ``` +//! 'help' will list available commands. When the debug server is enabled, the compiler tracks a lot +//! of extra state (see `Zcu.IncrementalDebugState`), so note that RSS will be higher than usual. + +comptime { + // This file should only be referenced when debug extensions are enabled. + std.debug.assert(@import("build_options").enable_debug_extensions); +} + +zcu: *Zcu, +thread: ?std.Thread, +running: std.atomic.Value(bool), +/// Held by our owner when an update is in-progress, and held by us when responding to a command. +/// So, essentially guards all access to `Compilation`, including `Zcu`. +mutex: std.Thread.Mutex, + +pub fn init(zcu: *Zcu) IncrementalDebugServer { + return .{ + .zcu = zcu, + .thread = null, + .running = .init(true), + .mutex = .{}, + }; +} + +pub fn deinit(ids: *IncrementalDebugServer) void { + if (ids.thread) |t| { + ids.running.store(false, .monotonic); + t.join(); + } +} + +const port = 7623; +pub fn spawn(ids: *IncrementalDebugServer) void { + std.debug.print("spawning incremental debug server on port {d}\n", .{port}); + ids.thread = std.Thread.spawn(.{ .allocator = ids.zcu.comp.arena }, runThread, .{ids}) catch |err| + std.process.fatal("failed to spawn incremental debug server: {s}", .{@errorName(err)}); +} +fn runThread(ids: *IncrementalDebugServer) void { + const gpa = ids.zcu.gpa; + + var cmd_buf: [1024]u8 = undefined; + var text_out: std.ArrayListUnmanaged(u8) = .empty; + defer text_out.deinit(gpa); + + const addr = std.net.Address.parseIp6("::", port) catch unreachable; + var server = addr.listen(.{}) catch @panic("IncrementalDebugServer: failed to listen"); + defer server.deinit(); + const conn = server.accept() catch @panic("IncrementalDebugServer: failed to accept"); + defer conn.stream.close(); + + while (ids.running.load(.monotonic)) { + conn.stream.writeAll("zig> ") catch @panic("IncrementalDebugServer: failed to write"); + var fbs = std.io.fixedBufferStream(&cmd_buf); + conn.stream.reader().streamUntilDelimiter(fbs.writer(), '\n', cmd_buf.len) catch |err| switch (err) { + error.EndOfStream => break, + else => @panic("IncrementalDebugServer: failed to read command"), + }; + const cmd_and_arg = std.mem.trim(u8, fbs.getWritten(), " \t\r\n"); + const cmd: []const u8, const arg: []const u8 = if (std.mem.indexOfScalar(u8, cmd_and_arg, ' ')) |i| + .{ cmd_and_arg[0..i], cmd_and_arg[i + 1 ..] } + else + .{ cmd_and_arg, "" }; + + text_out.clearRetainingCapacity(); + { + if (!ids.mutex.tryLock()) { + conn.stream.writeAll("waiting for in-progress update to finish...\n") catch @panic("IncrementalDebugServer: failed to write"); + ids.mutex.lock(); + } + defer ids.mutex.unlock(); + handleCommand(ids.zcu, &text_out, cmd, arg) catch @panic("IncrementalDebugServer: out of memory"); + } + text_out.append(gpa, '\n') catch @panic("IncrementalDebugServer: out of memory"); + conn.stream.writeAll(text_out.items) catch @panic("IncrementalDebugServer: failed to write"); + } + std.debug.print("closing incremental debug server\n", .{}); +} + +const help_str: []const u8 = + \\[str] arguments are any string. + \\[id] arguments are a numeric ID/index, like an InternPool index. + \\[unit] arguments are strings like 'func 1234' where '1234' is the relevant index (in this case an InternPool index). + \\ + \\MISC + \\ summary + \\ Dump some information about the whole ZCU. + \\ nav_info [id] + \\ Dump basic info about a NAV. + \\ + \\SEARCHING + \\ find_type [str] + \\ Find types (including dead ones) whose names contain the given substring. + \\ Starting with '^' or ending with '$' anchors to the start/end of the name. + \\ find_nav [str] + \\ Find NAVs (including dead ones) whose names contain the given substring. + \\ Starting with '^' or ending with '$' anchors to the start/end of the name. + \\ + \\UNITS + \\ unit_info [unit] + \\ Dump basic info about an analysis unit. + \\ unit_dependencies [unit] + \\ List all units which an analysis unit depends on. + \\ unit_trace [unit] + \\ Dump the current reference trace of an analysis unit. + \\ + \\TYPES + \\ type_info [id] + \\ Dump basic info about a type. + \\ type_namespace [id] + \\ List all declarations in the namespace of a type. + \\ +; + +fn handleCommand(zcu: *Zcu, output: *std.ArrayListUnmanaged(u8), cmd_str: []const u8, arg_str: []const u8) Allocator.Error!void { + const ip = &zcu.intern_pool; + const gpa = zcu.gpa; + const w = output.writer(gpa); + if (std.mem.eql(u8, cmd_str, "help")) { + try w.writeAll(help_str); + } else if (std.mem.eql(u8, cmd_str, "summary")) { + try w.print( + \\last generation: {d} + \\total container types: {d} + \\total NAVs: {d} + \\total units: {d} + \\ + , .{ + zcu.generation - 1, + zcu.incremental_debug_state.types.count(), + zcu.incremental_debug_state.navs.count(), + zcu.incremental_debug_state.units.count(), + }); + } else if (std.mem.eql(u8, cmd_str, "nav_info")) { + const nav_index: InternPool.Nav.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed nav index")); + const create_gen = zcu.incremental_debug_state.navs.get(nav_index) orelse return w.writeAll("unknown nav index"); + const nav = ip.getNav(nav_index); + try w.print( + \\name: '{}' + \\fqn: '{}' + \\status: {s} + \\created on generation: {d} + \\ + , .{ + nav.name.fmt(ip), + nav.fqn.fmt(ip), + @tagName(nav.status), + create_gen, + }); + switch (nav.status) { + .unresolved => {}, + .type_resolved, .fully_resolved => { + try w.writeAll("type: "); + try printType(.fromInterned(nav.typeOf(ip)), zcu, w); + try w.writeByte('\n'); + }, + } + } else if (std.mem.eql(u8, cmd_str, "find_type")) { + if (arg_str.len == 0) return w.writeAll("bad usage"); + const anchor_start = arg_str[0] == '^'; + const anchor_end = arg_str[arg_str.len - 1] == '$'; + const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)]; + var num_results: usize = 0; + for (zcu.incremental_debug_state.types.keys()) |type_ip_index| { + const ty: Type = .fromInterned(type_ip_index); + const ty_name = ty.containerTypeName(ip).toSlice(ip); + const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) { + 0b00 => std.mem.indexOf(u8, ty_name, query) != null, + 0b01 => std.mem.endsWith(u8, ty_name, query), + 0b10 => std.mem.startsWith(u8, ty_name, query), + 0b11 => std.mem.eql(u8, ty_name, query), + }; + if (success) { + num_results += 1; + try w.print("* type {d} ('{s}')\n", .{ @intFromEnum(type_ip_index), ty_name }); + } + } + try w.print("Found {d} results\n", .{num_results}); + } else if (std.mem.eql(u8, cmd_str, "find_nav")) { + if (arg_str.len == 0) return w.writeAll("bad usage"); + const anchor_start = arg_str[0] == '^'; + const anchor_end = arg_str[arg_str.len - 1] == '$'; + const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)]; + var num_results: usize = 0; + for (zcu.incremental_debug_state.navs.keys()) |nav_index| { + const nav = ip.getNav(nav_index); + const nav_fqn = nav.fqn.toSlice(ip); + const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) { + 0b00 => std.mem.indexOf(u8, nav_fqn, query) != null, + 0b01 => std.mem.endsWith(u8, nav_fqn, query), + 0b10 => std.mem.startsWith(u8, nav_fqn, query), + 0b11 => std.mem.eql(u8, nav_fqn, query), + }; + if (success) { + num_results += 1; + try w.print("* nav {d} ('{s}')\n", .{ @intFromEnum(nav_index), nav_fqn }); + } + } + try w.print("Found {d} results\n", .{num_results}); + } else if (std.mem.eql(u8, cmd_str, "unit_info")) { + const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit"); + const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit"); + var ref_str_buf: [32]u8 = undefined; + const ref_str: []const u8 = ref: { + const refs = try zcu.resolveReferences(); + const ref = refs.get(unit) orelse break :ref ""; + const referencer = (ref orelse break :ref "").referencer; + break :ref printAnalUnit(referencer, &ref_str_buf); + }; + const has_err: []const u8 = err: { + if (zcu.failed_analysis.contains(unit)) break :err "true"; + if (zcu.transitive_failed_analysis.contains(unit)) break :err "true (transitive)"; + break :err "false"; + }; + try w.print( + \\last update generation: {d} + \\current referencer: {s} + \\has error: {s} + \\ + , .{ + unit_info.last_update_gen, + ref_str, + has_err, + }); + } else if (std.mem.eql(u8, cmd_str, "unit_dependencies")) { + const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit"); + const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit"); + for (unit_info.deps.items, 0..) |dependee, i| { + try w.print("[{d}] ", .{i}); + switch (dependee) { + .src_hash, .namespace, .namespace_name, .zon_file, .embed_file => try w.print("{}", .{zcu.fmtDependee(dependee)}), + .nav_val, .nav_ty => |nav| try w.print("{s} {d}", .{ @tagName(dependee), @intFromEnum(nav) }), + .interned => |ip_index| switch (ip.indexToKey(ip_index)) { + .struct_type, .union_type, .enum_type => try w.print("type {d}", .{@intFromEnum(ip_index)}), + .func => try w.print("func {d}", .{@intFromEnum(ip_index)}), + else => unreachable, + }, + .memoized_state => |stage| try w.print("memoized_state {s}", .{@tagName(stage)}), + } + try w.writeByte('\n'); + } + } else if (std.mem.eql(u8, cmd_str, "unit_trace")) { + const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit"); + if (!zcu.incremental_debug_state.units.contains(unit)) return w.writeAll("unknown anal unit"); + const refs = try zcu.resolveReferences(); + if (!refs.contains(unit)) return w.writeAll("not referenced"); + var opt_cur: ?AnalUnit = unit; + while (opt_cur) |cur| { + var buf: [32]u8 = undefined; + try w.print("* {s}\n", .{printAnalUnit(cur, &buf)}); + opt_cur = if (refs.get(cur).?) |ref| ref.referencer else null; + } + } else if (std.mem.eql(u8, cmd_str, "type_info")) { + const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index")); + const create_gen = zcu.incremental_debug_state.types.get(ip_index) orelse return w.writeAll("unknown type"); + try w.print( + \\name: '{}' + \\created on generation: {d} + \\ + , .{ + Type.fromInterned(ip_index).containerTypeName(ip).fmt(ip), + create_gen, + }); + } else if (std.mem.eql(u8, cmd_str, "type_namespace")) { + const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index")); + if (!zcu.incremental_debug_state.types.contains(ip_index)) return w.writeAll("unknown type"); + const ns = zcu.namespacePtr(Type.fromInterned(ip_index).getNamespaceIndex(zcu)); + try w.print("{d} pub decls:\n", .{ns.pub_decls.count()}); + for (ns.pub_decls.keys()) |nav| { + try w.print("* nav {d}\n", .{@intFromEnum(nav)}); + } + try w.print("{d} non-pub decls:\n", .{ns.priv_decls.count()}); + for (ns.priv_decls.keys()) |nav| { + try w.print("* nav {d}\n", .{@intFromEnum(nav)}); + } + try w.print("{d} comptime decls:\n", .{ns.comptime_decls.items.len}); + for (ns.comptime_decls.items) |id| { + try w.print("* comptime {d}\n", .{@intFromEnum(id)}); + } + try w.print("{d} tests:\n", .{ns.test_decls.items.len}); + for (ns.test_decls.items) |nav| { + try w.print("* nav {d}\n", .{@intFromEnum(nav)}); + } + } else { + try w.writeAll("command not found; run 'help' for a command list"); + } +} + +fn parseIndex(str: []const u8) ?u32 { + return std.fmt.parseInt(u32, str, 10) catch null; +} +fn parseAnalUnit(str: []const u8) ?AnalUnit { + const split_idx = std.mem.indexOfScalar(u8, str, ' ') orelse return null; + const kind = str[0..split_idx]; + const idx_str = str[split_idx + 1 ..]; + if (std.mem.eql(u8, kind, "comptime")) { + return .wrap(.{ .@"comptime" = @enumFromInt(parseIndex(idx_str) orelse return null) }); + } else if (std.mem.eql(u8, kind, "nav_val")) { + return .wrap(.{ .nav_val = @enumFromInt(parseIndex(idx_str) orelse return null) }); + } else if (std.mem.eql(u8, kind, "nav_ty")) { + return .wrap(.{ .nav_ty = @enumFromInt(parseIndex(idx_str) orelse return null) }); + } else if (std.mem.eql(u8, kind, "type")) { + return .wrap(.{ .type = @enumFromInt(parseIndex(idx_str) orelse return null) }); + } else if (std.mem.eql(u8, kind, "func")) { + return .wrap(.{ .func = @enumFromInt(parseIndex(idx_str) orelse return null) }); + } else if (std.mem.eql(u8, kind, "memoized_state")) { + return .wrap(.{ .memoized_state = std.meta.stringToEnum( + InternPool.MemoizedStateStage, + idx_str, + ) orelse return null }); + } else { + return null; + } +} +fn printAnalUnit(unit: AnalUnit, buf: *[32]u8) []const u8 { + const idx: u32 = switch (unit.unwrap()) { + .memoized_state => |stage| return std.fmt.bufPrint(buf, "memoized_state {s}", .{@tagName(stage)}) catch unreachable, + inline else => |i| @intFromEnum(i), + }; + return std.fmt.bufPrint(buf, "{s} {d}", .{ @tagName(unit.unwrap()), idx }) catch unreachable; +} +fn printType(ty: Type, zcu: *const Zcu, w: anytype) !void { + const ip = &zcu.intern_pool; + switch (ip.indexToKey(ty.toIntern())) { + .int_type => |int| try w.print("{c}{d}", .{ + @as(u8, if (int.signedness == .unsigned) 'u' else 'i'), + int.bits, + }), + .tuple_type => try w.writeAll("(tuple)"), + .error_set_type => try w.writeAll("(error set)"), + .inferred_error_set_type => try w.writeAll("(inferred error set)"), + .func_type => try w.writeAll("(function)"), + .anyframe_type => try w.writeAll("(anyframe)"), + .vector_type => { + try w.print("@Vector({d}, ", .{ty.vectorLen(zcu)}); + try printType(ty.childType(zcu), zcu, w); + try w.writeByte(')'); + }, + .array_type => { + try w.print("[{d}]", .{ty.arrayLen(zcu)}); + try printType(ty.childType(zcu), zcu, w); + }, + .opt_type => { + try w.writeByte('?'); + try printType(ty.optionalChild(zcu), zcu, w); + }, + .error_union_type => { + try printType(ty.errorUnionSet(zcu), zcu, w); + try w.writeByte('!'); + try printType(ty.errorUnionPayload(zcu), zcu, w); + }, + .ptr_type => { + try w.writeAll("*(attrs) "); + try printType(ty.childType(zcu), zcu, w); + }, + .simple_type => |simple| try w.writeAll(@tagName(simple)), + + .struct_type, + .union_type, + .enum_type, + .opaque_type, + => try w.print("{}[{d}]", .{ ty.containerTypeName(ip).fmt(ip), @intFromEnum(ty.toIntern()) }), + + else => unreachable, + } +} + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const Compilation = @import("Compilation.zig"); +const Zcu = @import("Zcu.zig"); +const InternPool = @import("InternPool.zig"); +const Type = @import("Type.zig"); +const AnalUnit = InternPool.AnalUnit; + +const IncrementalDebugServer = @This(); diff --git a/src/Sema.zig b/src/Sema.zig index 2b7ffbe15d..60f857ace7 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2998,11 +2998,7 @@ fn zirStructDecl( errdefer pt.destroyNamespace(new_namespace_index); if (pt.zcu.comp.incremental) { - try ip.addDependency( - sema.gpa, - AnalUnit.wrap(.{ .type = wip_ty.index }), - .{ .src_hash = tracked_inst }, - ); + try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst }); } const decls = sema.code.bodySlice(extra_index, decls_len); @@ -3017,6 +3013,7 @@ fn zirStructDecl( } try sema.declareDependency(.{ .interned = wip_ty.index }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); } @@ -3247,6 +3244,7 @@ fn zirEnumDecl( // We've finished the initial construction of this type, and are about to perform analysis. // Set the namespace appropriately, and don't destroy anything on failure. + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); wip_ty.prepare(ip, new_namespace_index); done = true; @@ -3377,11 +3375,7 @@ fn zirUnionDecl( errdefer pt.destroyNamespace(new_namespace_index); if (pt.zcu.comp.incremental) { - try zcu.intern_pool.addDependency( - gpa, - AnalUnit.wrap(.{ .type = wip_ty.index }), - .{ .src_hash = tracked_inst }, - ); + try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst }); } const decls = sema.code.bodySlice(extra_index, decls_len); @@ -3396,6 +3390,7 @@ fn zirUnionDecl( } try sema.declareDependency(.{ .interned = wip_ty.index }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); } @@ -3481,6 +3476,7 @@ fn zirOpaqueDecl( try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index }); } try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); } @@ -8026,6 +8022,11 @@ fn analyzeCall( .generic_owner = func_val.?.toIntern(), .comptime_args = comptime_args, }); + if (zcu.comp.debugIncremental()) { + const nav = ip.indexToKey(func_instance).func.owner_nav; + const gop = try zcu.incremental_debug_state.navs.getOrPut(gpa, nav); + if (!gop.found_existing) gop.value_ptr.* = zcu.generation; + } // This call is problematic as it breaks guarantees about order-independency of semantic analysis. // These guarantees are necessary for incremental compilation and parallel semantic analysis. @@ -20345,6 +20346,7 @@ fn structInitAnon( if (block.ownerModule().strip) break :codegen_type; try zcu.comp.queueJob(.{ .codegen_type = wip.index }); } + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip.index); break :ty wip.finish(ip, new_namespace_index); }, .existing => |ty| ty, @@ -21406,6 +21408,7 @@ fn zirReify( }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); }, .@"union" => { @@ -21611,6 +21614,7 @@ fn reifyEnum( try sema.declareDependency(.{ .interned = wip_ty.index }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); wip_ty.prepare(ip, new_namespace_index); wip_ty.setTagTy(ip, tag_ty.toIntern()); done = true; @@ -21920,6 +21924,7 @@ fn reifyUnion( } try sema.declareDependency(.{ .interned = wip_ty.index }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); } @@ -22273,6 +22278,7 @@ fn reifyStruct( } try sema.declareDependency(.{ .interned = wip_ty.index }); try sema.addTypeReferenceEntry(src, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return Air.internedToRef(wip_ty.finish(ip, new_namespace_index)); } @@ -37485,8 +37491,8 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool } pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void { - const zcu = sema.pt.zcu; - if (!zcu.comp.incremental) return; + const pt = sema.pt; + if (!pt.zcu.comp.incremental) return; const gop = try sema.dependencies.getOrPut(sema.gpa, dependee); if (gop.found_existing) return; @@ -37508,7 +37514,7 @@ pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void { else => {}, } - try zcu.intern_pool.addDependency(sema.gpa, sema.owner, dependee); + try pt.addDependency(sema.owner, dependee); } fn isComptimeMutablePtr(sema: *Sema, val: Value) bool { @@ -37905,6 +37911,11 @@ pub fn resolveDeclaredEnum( }; defer sema.deinit(); + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, sema.owner); + info.last_update_gen = zcu.generation; + } + try sema.declareDependency(.{ .src_hash = tracked_inst }); var block: Block = .{ diff --git a/src/Type.zig b/src/Type.zig index ba64fb633a..b2a4b228d9 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -3797,6 +3797,11 @@ fn resolveStructInner( return error.AnalysisFail; } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner); + info.last_update_gen = zcu.generation; + } + var analysis_arena = std.heap.ArenaAllocator.init(gpa); defer analysis_arena.deinit(); @@ -3851,6 +3856,11 @@ fn resolveUnionInner( return error.AnalysisFail; } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner); + info.last_update_gen = zcu.generation; + } + var analysis_arena = std.heap.ArenaAllocator.init(gpa); defer analysis_arena.deinit(); diff --git a/src/Zcu.zig b/src/Zcu.zig index cbcac1149b..c70ef38d31 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -308,8 +308,56 @@ free_type_references: std.ArrayListUnmanaged(u32) = .empty, /// Populated by analysis of `AnalUnit.wrap(.{ .memoized_state = s })`, where `s` depends on the element. builtin_decl_values: BuiltinDecl.Memoized = .initFill(.none), +incremental_debug_state: if (build_options.enable_debug_extensions) IncrementalDebugState else void = + if (build_options.enable_debug_extensions) .init else {}, + generation: u32 = 0, +pub const IncrementalDebugState = struct { + /// All container types in the ZCU, even dead ones. + /// Value is the generation the type was created on. + types: std.AutoArrayHashMapUnmanaged(InternPool.Index, u32), + /// All `Nav`s in the ZCU, even dead ones. + /// Value is the generation the `Nav` was created on. + navs: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, u32), + /// All `AnalUnit`s in the ZCU, even dead ones. + units: std.AutoArrayHashMapUnmanaged(AnalUnit, UnitInfo), + + pub const init: IncrementalDebugState = .{ + .types = .empty, + .navs = .empty, + .units = .empty, + }; + pub fn deinit(ids: *IncrementalDebugState, gpa: Allocator) void { + for (ids.units.values()) |*unit_info| { + unit_info.deps.deinit(gpa); + } + ids.types.deinit(gpa); + ids.navs.deinit(gpa); + ids.units.deinit(gpa); + } + + pub const UnitInfo = struct { + last_update_gen: u32, + /// This information isn't easily recoverable from `InternPool`'s dependency storage format. + deps: std.ArrayListUnmanaged(InternPool.Dependee), + }; + pub fn getUnitInfo(ids: *IncrementalDebugState, gpa: Allocator, unit: AnalUnit) Allocator.Error!*UnitInfo { + const gop = try ids.units.getOrPut(gpa, unit); + if (!gop.found_existing) gop.value_ptr.* = .{ + .last_update_gen = std.math.maxInt(u32), + .deps = .empty, + }; + return gop.value_ptr; + } + pub fn newType(ids: *IncrementalDebugState, zcu: *Zcu, ty: InternPool.Index) Allocator.Error!void { + try ids.types.putNoClobber(zcu.gpa, ty, zcu.generation); + } + pub fn newNav(ids: *IncrementalDebugState, zcu: *Zcu, nav: InternPool.Nav.Index) Allocator.Error!void { + try ids.navs.putNoClobber(zcu.gpa, nav, zcu.generation); + } +}; + pub const PerThread = @import("Zcu/PerThread.zig"); pub const ImportTableAdapter = struct { @@ -2746,6 +2794,10 @@ pub fn deinit(zcu: *Zcu) void { zcu.free_type_references.deinit(gpa); if (zcu.resolved_references) |*r| r.deinit(gpa); + + if (zcu.comp.debugIncremental()) { + zcu.incremental_debug_state.deinit(gpa); + } } zcu.intern_pool.deinit(gpa); } diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index de855fa912..e32d99d162 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -635,6 +635,12 @@ pub fn ensureMemoizedStateUpToDate(pt: Zcu.PerThread, stage: InternPool.Memoized if (zcu.builtin_decl_values.get(to_check) != .none) return; } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + const any_changed: bool, const new_failed: bool = if (pt.analyzeMemoizedState(stage)) |any_changed| .{ any_changed or prev_failed, false } else |err| switch (err) { @@ -784,6 +790,12 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU return; } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + const unit_prog_node = zcu.sema_prog_node.start("comptime", 0); defer unit_prog_node.end(); @@ -958,6 +970,12 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu } } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0); defer unit_prog_node.end(); @@ -1331,6 +1349,12 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc } } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0); defer unit_prog_node.end(); @@ -1564,6 +1588,12 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, maybe_coerced_func_index: Inter if (func.analysisUnordered(ip).is_analyzed) return; } + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + const func_prog_node = zcu.sema_prog_node.start(ip.getNav(func.owner_nav).fqn.toSlice(ip), 0); defer func_prog_node.end(); @@ -1816,11 +1846,7 @@ fn createFileRootStruct( ip.namespacePtr(namespace_index).owner_type = wip_ty.index; if (zcu.comp.incremental) { - try ip.addDependency( - gpa, - .wrap(.{ .type = wip_ty.index }), - .{ .src_hash = tracked_inst }, - ); + try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst }); } try pt.scanNamespace(namespace_index, decls); @@ -1832,6 +1858,7 @@ fn createFileRootStruct( try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index }); } zcu.setFileRootType(file_index, wip_ty.index); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return wip_ty.finish(ip, namespace_index); } @@ -2734,10 +2761,11 @@ const ScanDeclIter = struct { else => unit: { const name = maybe_name.unwrap().?; const fqn = try namespace.internFullyQualifiedName(ip, gpa, pt.tid, name); - const nav = if (existing_unit) |eu| - eu.unwrap().nav_val - else - try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace"); + const nav = if (existing_unit) |eu| eu.unwrap().nav_val else nav: { + const nav = try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace"); + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newNav(zcu, nav); + break :nav nav; + }; const unit: AnalUnit = .wrap(.{ .nav_val = nav }); @@ -3911,6 +3939,7 @@ pub fn getExtern(pt: Zcu.PerThread, key: InternPool.Key.Extern) Allocator.Error! if (result.new_nav.unwrap()) |nav| { // This job depends on any resolve_type_fully jobs queued up before it. try pt.zcu.comp.queueJob(.{ .codegen_nav = nav }); + if (pt.zcu.comp.debugIncremental()) try pt.zcu.incremental_debug_state.newNav(pt.zcu, nav); } return result.index; } @@ -3979,6 +4008,12 @@ pub fn ensureTypeUpToDate(pt: Zcu.PerThread, ty: InternPool.Index) Zcu.SemaError _ = zcu.transitive_failed_analysis.swapRemove(anal_unit); zcu.intern_pool.removeDependenciesForDepender(gpa, anal_unit); + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit); + info.last_update_gen = zcu.generation; + info.deps.clearRetainingCapacity(); + } + switch (ip.indexToKey(ty)) { .struct_type => return pt.recreateStructType(ty, declared_ty_key), .union_type => return pt.recreateUnionType(ty, declared_ty_key), @@ -4042,11 +4077,7 @@ fn recreateStructType( errdefer wip_ty.cancel(ip, pt.tid); wip_ty.setName(ip, struct_obj.name); - try ip.addDependency( - gpa, - .wrap(.{ .type = wip_ty.index }), - .{ .src_hash = key.zir_index }, - ); + try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index }); zcu.namespacePtr(struct_obj.namespace).owner_type = wip_ty.index; // No need to re-scan the namespace -- `zirStructDecl` will ultimately do that if the type is still alive. try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index }); @@ -4058,6 +4089,7 @@ fn recreateStructType( try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index }); } + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); const new_ty = wip_ty.finish(ip, struct_obj.namespace); if (inst_info.inst == .main_struct_inst) { // This is the root type of a file! Update the reference. @@ -4138,11 +4170,7 @@ fn recreateUnionType( errdefer wip_ty.cancel(ip, pt.tid); wip_ty.setName(ip, union_obj.name); - try ip.addDependency( - gpa, - .wrap(.{ .type = wip_ty.index }), - .{ .src_hash = key.zir_index }, - ); + try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index }); zcu.namespacePtr(namespace_index).owner_type = wip_ty.index; // No need to re-scan the namespace -- `zirUnionDecl` will ultimately do that if the type is still alive. try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index }); @@ -4154,6 +4182,7 @@ fn recreateUnionType( try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index }); } + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); return wip_ty.finish(ip, namespace_index); } @@ -4255,6 +4284,7 @@ fn recreateEnumType( zcu.namespacePtr(namespace_index).owner_type = wip_ty.index; // No need to re-scan the namespace -- `zirEnumDecl` will ultimately do that if the type is still alive. + if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index); wip_ty.prepare(ip, namespace_index); done = true; @@ -4432,3 +4462,13 @@ pub fn refValue(pt: Zcu.PerThread, val: InternPool.Index) Zcu.SemaError!InternPo .byte_offset = 0, } }); } + +pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dependee) Allocator.Error!void { + const zcu = pt.zcu; + const gpa = zcu.gpa; + try zcu.intern_pool.addDependency(gpa, unit, dependee); + if (zcu.comp.debugIncremental()) { + const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit); + try info.deps.append(gpa, dependee); + } +} diff --git a/src/main.zig b/src/main.zig index f704ce4bfe..d0006c3db3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -677,6 +677,7 @@ const usage_build_generic = \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format \\ --debug-rt Debug compiler runtime libraries + \\ --debug-incremental Enable incremental compilation debug features \\ ; @@ -832,6 +833,7 @@ fn buildOutputType( var data_sections = false; var listen: Listen = .none; var debug_compile_errors = false; + var debug_incremental = false; var verbose_link = (native_os != .wasi or builtin.link_libc) and EnvVar.ZIG_VERBOSE_LINK.isSet(); var verbose_cc = (native_os != .wasi or builtin.link_libc) and @@ -1383,6 +1385,12 @@ fn buildOutputType( } } else if (mem.eql(u8, arg, "--debug-rt")) { debug_compiler_runtime_libs = true; + } else if (mem.eql(u8, arg, "--debug-incremental")) { + if (build_options.enable_debug_extensions) { + debug_incremental = true; + } else { + warn("Zig was compiled without debug extensions. --debug-incremental has no effect.", .{}); + } } else if (mem.eql(u8, arg, "-fincremental")) { dev.check(.incremental); opt_incremental = true; @@ -3460,6 +3468,9 @@ fn buildOutputType( }; const incremental = opt_incremental orelse false; + if (debug_incremental and !incremental) { + fatal("--debug-incremental requires -fincremental", .{}); + } const disable_lld_caching = !output_to_cache; @@ -3592,6 +3603,7 @@ fn buildOutputType( .cache_mode = cache_mode, .subsystem = subsystem, .debug_compile_errors = debug_compile_errors, + .debug_incremental = debug_incremental, .incremental = incremental, .enable_link_snapshots = enable_link_snapshots, .install_name = install_name, @@ -4195,9 +4207,25 @@ fn serve( const main_progress_node = std.Progress.start(.{}); const file_system_inputs = comp.file_system_inputs.?; + const IncrementalDebugServer = if (build_options.enable_debug_extensions) + @import("IncrementalDebugServer.zig") + else + void; + + var ids: IncrementalDebugServer = if (comp.debugIncremental()) ids: { + break :ids .init(comp.zcu orelse @panic("--debug-incremental requires a ZCU")); + } else undefined; + defer if (comp.debugIncremental()) ids.deinit(); + + if (comp.debugIncremental()) ids.spawn(); + while (true) { const hdr = try server.receiveMessage(); + // Lock the debug server while hanling the message. + if (comp.debugIncremental()) ids.mutex.lock(); + defer if (comp.debugIncremental()) ids.mutex.unlock(); + switch (hdr.tag) { .exit => return cleanExit(), .update => {