compiler: introduce incremental debug server

In a compiler built with debug extensions, pass `--debug-incremental` to
spawn the "incremental debug server". This is a TCP server exposing a
REPL which allows querying a bunch of compiler state, some of which is
stored only when that flag is passed. Eventually, this will probably
move into `std.zig.Server`/`std.zig.Client`, but this is easier to work
with right now. The easiest way to interact with the server is `telnet`.

lib/compiler/build_runner.zig

@@ -236,6 +236,8 @@ pub fn main() !void {
                 graph.debug_compiler_runtime_libs = true;
             } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
                 builder.debug_compile_errors = true;
+            } else if (mem.eql(u8, arg, "--debug-incremental")) {
+                builder.debug_incremental = true;
             } else if (mem.eql(u8, arg, "--system")) {
                 // The usage text shows another argument after this parameter
                 // but it is handled by the parent process. The build runner

lib/std/Build.zig

@@ -59,6 +59,7 @@ pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
 args: ?[]const []const u8 = null,
 debug_log_scopes: []const []const u8 = &.{},
 debug_compile_errors: bool = false,
+debug_incremental: bool = false,
 debug_pkg_config: bool = false,
 /// Number of stack frames captured when a `StackTrace` is recorded for debug purposes,
 /// in particular at `Step` creation.
@@ -385,6 +386,7 @@ fn createChildOnly(
         .cache_root = parent.cache_root,
         .debug_log_scopes = parent.debug_log_scopes,
         .debug_compile_errors = parent.debug_compile_errors,
+        .debug_incremental = parent.debug_incremental,
         .debug_pkg_config = parent.debug_pkg_config,
         .enable_darling = parent.enable_darling,
         .enable_qemu = parent.enable_qemu,

lib/std/Build/Step/Compile.zig

@@ -1447,6 +1447,10 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
         try zig_args.append("--debug-compile-errors");
     }
 
+    if (b.debug_incremental) {
+        try zig_args.append("--debug-incremental");
+    }
+
     if (b.verbose_cimport) try zig_args.append("--verbose-cimport");
     if (b.verbose_air) try zig_args.append("--verbose-air");
     if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));

src/Compilation.zig

@@ -190,6 +190,8 @@ time_report: bool,
 stack_report: bool,
 debug_compiler_runtime_libs: bool,
 debug_compile_errors: bool,
+/// Do not check this field directly. Instead, use the `debugIncremental` wrapper function.
+debug_incremental: bool,
 incremental: bool,
 alloc_failure_occurred: bool = false,
 last_update_was_cache_hit: bool = false,
@@ -768,6 +770,14 @@ pub const Directories = struct {
     }
 };
 
+/// This small wrapper function just checks whether debug extensions are enabled before checking
+/// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller,
+/// preventing debugging features from making it into release builds of the compiler.
+pub inline fn debugIncremental(comp: *const Compilation) bool {
+    if (!build_options.enable_debug_extensions) return false;
+    return comp.debug_incremental;
+}
+
 pub const default_stack_protector_buffer_size = target_util.default_stack_protector_buffer_size;
 pub const SemaError = Zcu.SemaError;
 
@@ -1598,6 +1608,7 @@ pub const CreateOptions = struct {
     verbose_llvm_cpu_features: bool = false,
     debug_compiler_runtime_libs: bool = false,
     debug_compile_errors: bool = false,
+    debug_incremental: bool = false,
     incremental: bool = false,
     /// Normally when you create a `Compilation`, Zig will automatically build
     /// and link in required dependencies, such as compiler-rt and libc. When
@@ -1968,6 +1979,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
             .test_name_prefix = options.test_name_prefix,
             .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs,
             .debug_compile_errors = options.debug_compile_errors,
+            .debug_incremental = options.debug_incremental,
             .incremental = options.incremental,
             .root_name = root_name,
             .sysroot = sysroot,

src/IncrementalDebugServer.zig

@@ -0,0 +1,383 @@
+//! This is a simple TCP server which exposes a REPL useful for debugging incremental compilation
+//! issues. Eventually, this logic should move into `std.zig.Client`/`std.zig.Server` or something
+//! similar, but for now, this works. The server is enabled by the '--debug-incremental' CLI flag.
+//! The easiest way to interact with the REPL is to use `telnet`:
+//! ```
+//! telnet "::1" 7623
+//! ```
+//! 'help' will list available commands. When the debug server is enabled, the compiler tracks a lot
+//! of extra state (see `Zcu.IncrementalDebugState`), so note that RSS will be higher than usual.
+
+comptime {
+    // This file should only be referenced when debug extensions are enabled.
+    std.debug.assert(@import("build_options").enable_debug_extensions);
+}
+
+zcu: *Zcu,
+thread: ?std.Thread,
+running: std.atomic.Value(bool),
+/// Held by our owner when an update is in-progress, and held by us when responding to a command.
+/// So, essentially guards all access to `Compilation`, including `Zcu`.
+mutex: std.Thread.Mutex,
+
+pub fn init(zcu: *Zcu) IncrementalDebugServer {
+    return .{
+        .zcu = zcu,
+        .thread = null,
+        .running = .init(true),
+        .mutex = .{},
+    };
+}
+
+pub fn deinit(ids: *IncrementalDebugServer) void {
+    if (ids.thread) |t| {
+        ids.running.store(false, .monotonic);
+        t.join();
+    }
+}
+
+const port = 7623;
+pub fn spawn(ids: *IncrementalDebugServer) void {
+    std.debug.print("spawning incremental debug server on port {d}\n", .{port});
+    ids.thread = std.Thread.spawn(.{ .allocator = ids.zcu.comp.arena }, runThread, .{ids}) catch |err|
+        std.process.fatal("failed to spawn incremental debug server: {s}", .{@errorName(err)});
+}
+fn runThread(ids: *IncrementalDebugServer) void {
+    const gpa = ids.zcu.gpa;
+
+    var cmd_buf: [1024]u8 = undefined;
+    var text_out: std.ArrayListUnmanaged(u8) = .empty;
+    defer text_out.deinit(gpa);
+
+    const addr = std.net.Address.parseIp6("::", port) catch unreachable;
+    var server = addr.listen(.{}) catch @panic("IncrementalDebugServer: failed to listen");
+    defer server.deinit();
+    const conn = server.accept() catch @panic("IncrementalDebugServer: failed to accept");
+    defer conn.stream.close();
+
+    while (ids.running.load(.monotonic)) {
+        conn.stream.writeAll("zig> ") catch @panic("IncrementalDebugServer: failed to write");
+        var fbs = std.io.fixedBufferStream(&cmd_buf);
+        conn.stream.reader().streamUntilDelimiter(fbs.writer(), '\n', cmd_buf.len) catch |err| switch (err) {
+            error.EndOfStream => break,
+            else => @panic("IncrementalDebugServer: failed to read command"),
+        };
+        const cmd_and_arg = std.mem.trim(u8, fbs.getWritten(), " \t\r\n");
+        const cmd: []const u8, const arg: []const u8 = if (std.mem.indexOfScalar(u8, cmd_and_arg, ' ')) |i|
+            .{ cmd_and_arg[0..i], cmd_and_arg[i + 1 ..] }
+        else
+            .{ cmd_and_arg, "" };
+
+        text_out.clearRetainingCapacity();
+        {
+            if (!ids.mutex.tryLock()) {
+                conn.stream.writeAll("waiting for in-progress update to finish...\n") catch @panic("IncrementalDebugServer: failed to write");
+                ids.mutex.lock();
+            }
+            defer ids.mutex.unlock();
+            handleCommand(ids.zcu, &text_out, cmd, arg) catch @panic("IncrementalDebugServer: out of memory");
+        }
+        text_out.append(gpa, '\n') catch @panic("IncrementalDebugServer: out of memory");
+        conn.stream.writeAll(text_out.items) catch @panic("IncrementalDebugServer: failed to write");
+    }
+    std.debug.print("closing incremental debug server\n", .{});
+}
+
+const help_str: []const u8 =
+    \\[str] arguments are any string.
+    \\[id] arguments are a numeric ID/index, like an InternPool index.
+    \\[unit] arguments are strings like 'func 1234' where '1234' is the relevant index (in this case an InternPool index).
+    \\
+    \\MISC
+    \\  summary
+    \\    Dump some information about the whole ZCU.
+    \\  nav_info [id]
+    \\    Dump basic info about a NAV.
+    \\
+    \\SEARCHING
+    \\  find_type [str]
+    \\    Find types (including dead ones) whose names contain the given substring.
+    \\    Starting with '^' or ending with '$' anchors to the start/end of the name.
+    \\  find_nav [str]
+    \\    Find NAVs (including dead ones) whose names contain the given substring.
+    \\    Starting with '^' or ending with '$' anchors to the start/end of the name.
+    \\
+    \\UNITS
+    \\  unit_info [unit]
+    \\    Dump basic info about an analysis unit.
+    \\  unit_dependencies [unit]
+    \\    List all units which an analysis unit depends on.
+    \\  unit_trace [unit]
+    \\    Dump the current reference trace of an analysis unit.
+    \\
+    \\TYPES
+    \\  type_info [id]
+    \\    Dump basic info about a type.
+    \\  type_namespace [id]
+    \\    List all declarations in the namespace of a type.
+    \\
+;
+
+fn handleCommand(zcu: *Zcu, output: *std.ArrayListUnmanaged(u8), cmd_str: []const u8, arg_str: []const u8) Allocator.Error!void {
+    const ip = &zcu.intern_pool;
+    const gpa = zcu.gpa;
+    const w = output.writer(gpa);
+    if (std.mem.eql(u8, cmd_str, "help")) {
+        try w.writeAll(help_str);
+    } else if (std.mem.eql(u8, cmd_str, "summary")) {
+        try w.print(
+            \\last generation: {d}
+            \\total container types: {d}
+            \\total NAVs: {d}
+            \\total units: {d}
+            \\
+        , .{
+            zcu.generation - 1,
+            zcu.incremental_debug_state.types.count(),
+            zcu.incremental_debug_state.navs.count(),
+            zcu.incremental_debug_state.units.count(),
+        });
+    } else if (std.mem.eql(u8, cmd_str, "nav_info")) {
+        const nav_index: InternPool.Nav.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed nav index"));
+        const create_gen = zcu.incremental_debug_state.navs.get(nav_index) orelse return w.writeAll("unknown nav index");
+        const nav = ip.getNav(nav_index);
+        try w.print(
+            \\name: '{}'
+            \\fqn: '{}'
+            \\status: {s}
+            \\created on generation: {d}
+            \\
+        , .{
+            nav.name.fmt(ip),
+            nav.fqn.fmt(ip),
+            @tagName(nav.status),
+            create_gen,
+        });
+        switch (nav.status) {
+            .unresolved => {},
+            .type_resolved, .fully_resolved => {
+                try w.writeAll("type: ");
+                try printType(.fromInterned(nav.typeOf(ip)), zcu, w);
+                try w.writeByte('\n');
+            },
+        }
+    } else if (std.mem.eql(u8, cmd_str, "find_type")) {
+        if (arg_str.len == 0) return w.writeAll("bad usage");
+        const anchor_start = arg_str[0] == '^';
+        const anchor_end = arg_str[arg_str.len - 1] == '$';
+        const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)];
+        var num_results: usize = 0;
+        for (zcu.incremental_debug_state.types.keys()) |type_ip_index| {
+            const ty: Type = .fromInterned(type_ip_index);
+            const ty_name = ty.containerTypeName(ip).toSlice(ip);
+            const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) {
+                0b00 => std.mem.indexOf(u8, ty_name, query) != null,
+                0b01 => std.mem.endsWith(u8, ty_name, query),
+                0b10 => std.mem.startsWith(u8, ty_name, query),
+                0b11 => std.mem.eql(u8, ty_name, query),
+            };
+            if (success) {
+                num_results += 1;
+                try w.print("* type {d} ('{s}')\n", .{ @intFromEnum(type_ip_index), ty_name });
+            }
+        }
+        try w.print("Found {d} results\n", .{num_results});
+    } else if (std.mem.eql(u8, cmd_str, "find_nav")) {
+        if (arg_str.len == 0) return w.writeAll("bad usage");
+        const anchor_start = arg_str[0] == '^';
+        const anchor_end = arg_str[arg_str.len - 1] == '$';
+        const query = arg_str[@intFromBool(anchor_start) .. arg_str.len - @intFromBool(anchor_end)];
+        var num_results: usize = 0;
+        for (zcu.incremental_debug_state.navs.keys()) |nav_index| {
+            const nav = ip.getNav(nav_index);
+            const nav_fqn = nav.fqn.toSlice(ip);
+            const success = switch (@as(u2, @intFromBool(anchor_start)) << 1 | @intFromBool(anchor_end)) {
+                0b00 => std.mem.indexOf(u8, nav_fqn, query) != null,
+                0b01 => std.mem.endsWith(u8, nav_fqn, query),
+                0b10 => std.mem.startsWith(u8, nav_fqn, query),
+                0b11 => std.mem.eql(u8, nav_fqn, query),
+            };
+            if (success) {
+                num_results += 1;
+                try w.print("* nav {d} ('{s}')\n", .{ @intFromEnum(nav_index), nav_fqn });
+            }
+        }
+        try w.print("Found {d} results\n", .{num_results});
+    } else if (std.mem.eql(u8, cmd_str, "unit_info")) {
+        const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
+        const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit");
+        var ref_str_buf: [32]u8 = undefined;
+        const ref_str: []const u8 = ref: {
+            const refs = try zcu.resolveReferences();
+            const ref = refs.get(unit) orelse break :ref "<unreferenced>";
+            const referencer = (ref orelse break :ref "<analysis root>").referencer;
+            break :ref printAnalUnit(referencer, &ref_str_buf);
+        };
+        const has_err: []const u8 = err: {
+            if (zcu.failed_analysis.contains(unit)) break :err "true";
+            if (zcu.transitive_failed_analysis.contains(unit)) break :err "true (transitive)";
+            break :err "false";
+        };
+        try w.print(
+            \\last update generation: {d}
+            \\current referencer: {s}
+            \\has error: {s}
+            \\
+        , .{
+            unit_info.last_update_gen,
+            ref_str,
+            has_err,
+        });
+    } else if (std.mem.eql(u8, cmd_str, "unit_dependencies")) {
+        const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
+        const unit_info = zcu.incremental_debug_state.units.get(unit) orelse return w.writeAll("unknown anal unit");
+        for (unit_info.deps.items, 0..) |dependee, i| {
+            try w.print("[{d}] ", .{i});
+            switch (dependee) {
+                .src_hash, .namespace, .namespace_name, .zon_file, .embed_file => try w.print("{}", .{zcu.fmtDependee(dependee)}),
+                .nav_val, .nav_ty => |nav| try w.print("{s} {d}", .{ @tagName(dependee), @intFromEnum(nav) }),
+                .interned => |ip_index| switch (ip.indexToKey(ip_index)) {
+                    .struct_type, .union_type, .enum_type => try w.print("type {d}", .{@intFromEnum(ip_index)}),
+                    .func => try w.print("func {d}", .{@intFromEnum(ip_index)}),
+                    else => unreachable,
+                },
+                .memoized_state => |stage| try w.print("memoized_state {s}", .{@tagName(stage)}),
+            }
+            try w.writeByte('\n');
+        }
+    } else if (std.mem.eql(u8, cmd_str, "unit_trace")) {
+        const unit = parseAnalUnit(arg_str) orelse return w.writeAll("malformed anal unit");
+        if (!zcu.incremental_debug_state.units.contains(unit)) return w.writeAll("unknown anal unit");
+        const refs = try zcu.resolveReferences();
+        if (!refs.contains(unit)) return w.writeAll("not referenced");
+        var opt_cur: ?AnalUnit = unit;
+        while (opt_cur) |cur| {
+            var buf: [32]u8 = undefined;
+            try w.print("* {s}\n", .{printAnalUnit(cur, &buf)});
+            opt_cur = if (refs.get(cur).?) |ref| ref.referencer else null;
+        }
+    } else if (std.mem.eql(u8, cmd_str, "type_info")) {
+        const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index"));
+        const create_gen = zcu.incremental_debug_state.types.get(ip_index) orelse return w.writeAll("unknown type");
+        try w.print(
+            \\name: '{}'
+            \\created on generation: {d}
+            \\
+        , .{
+            Type.fromInterned(ip_index).containerTypeName(ip).fmt(ip),
+            create_gen,
+        });
+    } else if (std.mem.eql(u8, cmd_str, "type_namespace")) {
+        const ip_index: InternPool.Index = @enumFromInt(parseIndex(arg_str) orelse return w.writeAll("malformed ip index"));
+        if (!zcu.incremental_debug_state.types.contains(ip_index)) return w.writeAll("unknown type");
+        const ns = zcu.namespacePtr(Type.fromInterned(ip_index).getNamespaceIndex(zcu));
+        try w.print("{d} pub decls:\n", .{ns.pub_decls.count()});
+        for (ns.pub_decls.keys()) |nav| {
+            try w.print("* nav {d}\n", .{@intFromEnum(nav)});
+        }
+        try w.print("{d} non-pub decls:\n", .{ns.priv_decls.count()});
+        for (ns.priv_decls.keys()) |nav| {
+            try w.print("* nav {d}\n", .{@intFromEnum(nav)});
+        }
+        try w.print("{d} comptime decls:\n", .{ns.comptime_decls.items.len});
+        for (ns.comptime_decls.items) |id| {
+            try w.print("* comptime {d}\n", .{@intFromEnum(id)});
+        }
+        try w.print("{d} tests:\n", .{ns.test_decls.items.len});
+        for (ns.test_decls.items) |nav| {
+            try w.print("* nav {d}\n", .{@intFromEnum(nav)});
+        }
+    } else {
+        try w.writeAll("command not found; run 'help' for a command list");
+    }
+}
+
+fn parseIndex(str: []const u8) ?u32 {
+    return std.fmt.parseInt(u32, str, 10) catch null;
+}
+fn parseAnalUnit(str: []const u8) ?AnalUnit {
+    const split_idx = std.mem.indexOfScalar(u8, str, ' ') orelse return null;
+    const kind = str[0..split_idx];
+    const idx_str = str[split_idx + 1 ..];
+    if (std.mem.eql(u8, kind, "comptime")) {
+        return .wrap(.{ .@"comptime" = @enumFromInt(parseIndex(idx_str) orelse return null) });
+    } else if (std.mem.eql(u8, kind, "nav_val")) {
+        return .wrap(.{ .nav_val = @enumFromInt(parseIndex(idx_str) orelse return null) });
+    } else if (std.mem.eql(u8, kind, "nav_ty")) {
+        return .wrap(.{ .nav_ty = @enumFromInt(parseIndex(idx_str) orelse return null) });
+    } else if (std.mem.eql(u8, kind, "type")) {
+        return .wrap(.{ .type = @enumFromInt(parseIndex(idx_str) orelse return null) });
+    } else if (std.mem.eql(u8, kind, "func")) {
+        return .wrap(.{ .func = @enumFromInt(parseIndex(idx_str) orelse return null) });
+    } else if (std.mem.eql(u8, kind, "memoized_state")) {
+        return .wrap(.{ .memoized_state = std.meta.stringToEnum(
+            InternPool.MemoizedStateStage,
+            idx_str,
+        ) orelse return null });
+    } else {
+        return null;
+    }
+}
+fn printAnalUnit(unit: AnalUnit, buf: *[32]u8) []const u8 {
+    const idx: u32 = switch (unit.unwrap()) {
+        .memoized_state => |stage| return std.fmt.bufPrint(buf, "memoized_state {s}", .{@tagName(stage)}) catch unreachable,
+        inline else => |i| @intFromEnum(i),
+    };
+    return std.fmt.bufPrint(buf, "{s} {d}", .{ @tagName(unit.unwrap()), idx }) catch unreachable;
+}
+fn printType(ty: Type, zcu: *const Zcu, w: anytype) !void {
+    const ip = &zcu.intern_pool;
+    switch (ip.indexToKey(ty.toIntern())) {
+        .int_type => |int| try w.print("{c}{d}", .{
+            @as(u8, if (int.signedness == .unsigned) 'u' else 'i'),
+            int.bits,
+        }),
+        .tuple_type => try w.writeAll("(tuple)"),
+        .error_set_type => try w.writeAll("(error set)"),
+        .inferred_error_set_type => try w.writeAll("(inferred error set)"),
+        .func_type => try w.writeAll("(function)"),
+        .anyframe_type => try w.writeAll("(anyframe)"),
+        .vector_type => {
+            try w.print("@Vector({d}, ", .{ty.vectorLen(zcu)});
+            try printType(ty.childType(zcu), zcu, w);
+            try w.writeByte(')');
+        },
+        .array_type => {
+            try w.print("[{d}]", .{ty.arrayLen(zcu)});
+            try printType(ty.childType(zcu), zcu, w);
+        },
+        .opt_type => {
+            try w.writeByte('?');
+            try printType(ty.optionalChild(zcu), zcu, w);
+        },
+        .error_union_type => {
+            try printType(ty.errorUnionSet(zcu), zcu, w);
+            try w.writeByte('!');
+            try printType(ty.errorUnionPayload(zcu), zcu, w);
+        },
+        .ptr_type => {
+            try w.writeAll("*(attrs) ");
+            try printType(ty.childType(zcu), zcu, w);
+        },
+        .simple_type => |simple| try w.writeAll(@tagName(simple)),
+
+        .struct_type,
+        .union_type,
+        .enum_type,
+        .opaque_type,
+        => try w.print("{}[{d}]", .{ ty.containerTypeName(ip).fmt(ip), @intFromEnum(ty.toIntern()) }),
+
+        else => unreachable,
+    }
+}
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+
+const Compilation = @import("Compilation.zig");
+const Zcu = @import("Zcu.zig");
+const InternPool = @import("InternPool.zig");
+const Type = @import("Type.zig");
+const AnalUnit = InternPool.AnalUnit;
+
+const IncrementalDebugServer = @This();

src/Sema.zig

@@ -2998,11 +2998,7 @@ fn zirStructDecl(
     errdefer pt.destroyNamespace(new_namespace_index);
 
     if (pt.zcu.comp.incremental) {
-        try ip.addDependency(
+        try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
-            sema.gpa,
-            AnalUnit.wrap(.{ .type = wip_ty.index }),
-            .{ .src_hash = tracked_inst },
-        );
     }
 
     const decls = sema.code.bodySlice(extra_index, decls_len);
@@ -3017,6 +3013,7 @@ fn zirStructDecl(
     }
     try sema.declareDependency(.{ .interned = wip_ty.index });
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
 }
 
@@ -3247,6 +3244,7 @@ fn zirEnumDecl(
 
     // We've finished the initial construction of this type, and are about to perform analysis.
     // Set the namespace appropriately, and don't destroy anything on failure.
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     wip_ty.prepare(ip, new_namespace_index);
     done = true;
 
@@ -3377,11 +3375,7 @@ fn zirUnionDecl(
     errdefer pt.destroyNamespace(new_namespace_index);
 
     if (pt.zcu.comp.incremental) {
-        try zcu.intern_pool.addDependency(
+        try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
-            gpa,
-            AnalUnit.wrap(.{ .type = wip_ty.index }),
-            .{ .src_hash = tracked_inst },
-        );
     }
 
     const decls = sema.code.bodySlice(extra_index, decls_len);
@@ -3396,6 +3390,7 @@ fn zirUnionDecl(
     }
     try sema.declareDependency(.{ .interned = wip_ty.index });
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
 }
 
@@ -3481,6 +3476,7 @@ fn zirOpaqueDecl(
         try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
     }
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
 }
 
@@ -8026,6 +8022,11 @@ fn analyzeCall(
                 .generic_owner = func_val.?.toIntern(),
                 .comptime_args = comptime_args,
             });
+            if (zcu.comp.debugIncremental()) {
+                const nav = ip.indexToKey(func_instance).func.owner_nav;
+                const gop = try zcu.incremental_debug_state.navs.getOrPut(gpa, nav);
+                if (!gop.found_existing) gop.value_ptr.* = zcu.generation;
+            }
 
             // This call is problematic as it breaks guarantees about order-independency of semantic analysis.
             // These guarantees are necessary for incremental compilation and parallel semantic analysis.
@@ -20345,6 +20346,7 @@ fn structInitAnon(
                 if (block.ownerModule().strip) break :codegen_type;
                 try zcu.comp.queueJob(.{ .codegen_type = wip.index });
             }
+            if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip.index);
             break :ty wip.finish(ip, new_namespace_index);
         },
         .existing => |ty| ty,
@@ -21406,6 +21408,7 @@ fn zirReify(
             });
 
             try sema.addTypeReferenceEntry(src, wip_ty.index);
+            if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
             return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
         },
         .@"union" => {
@@ -21611,6 +21614,7 @@ fn reifyEnum(
 
     try sema.declareDependency(.{ .interned = wip_ty.index });
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     wip_ty.prepare(ip, new_namespace_index);
     wip_ty.setTagTy(ip, tag_ty.toIntern());
     done = true;
@@ -21920,6 +21924,7 @@ fn reifyUnion(
     }
     try sema.declareDependency(.{ .interned = wip_ty.index });
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
 }
 
@@ -22273,6 +22278,7 @@ fn reifyStruct(
     }
     try sema.declareDependency(.{ .interned = wip_ty.index });
     try sema.addTypeReferenceEntry(src, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return Air.internedToRef(wip_ty.finish(ip, new_namespace_index));
 }
 
@@ -37485,8 +37491,8 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool
 }
 
 pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void {
-    const zcu = sema.pt.zcu;
+    const pt = sema.pt;
-    if (!zcu.comp.incremental) return;
+    if (!pt.zcu.comp.incremental) return;
 
     const gop = try sema.dependencies.getOrPut(sema.gpa, dependee);
     if (gop.found_existing) return;
@@ -37508,7 +37514,7 @@ pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void {
         else => {},
     }
 
-    try zcu.intern_pool.addDependency(sema.gpa, sema.owner, dependee);
+    try pt.addDependency(sema.owner, dependee);
 }
 
 fn isComptimeMutablePtr(sema: *Sema, val: Value) bool {
@@ -37905,6 +37911,11 @@ pub fn resolveDeclaredEnum(
     };
     defer sema.deinit();
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, sema.owner);
+        info.last_update_gen = zcu.generation;
+    }
+
     try sema.declareDependency(.{ .src_hash = tracked_inst });
 
     var block: Block = .{

src/Type.zig

@@ -3797,6 +3797,11 @@ fn resolveStructInner(
         return error.AnalysisFail;
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner);
+        info.last_update_gen = zcu.generation;
+    }
+
     var analysis_arena = std.heap.ArenaAllocator.init(gpa);
     defer analysis_arena.deinit();
 
@@ -3851,6 +3856,11 @@ fn resolveUnionInner(
         return error.AnalysisFail;
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner);
+        info.last_update_gen = zcu.generation;
+    }
+
     var analysis_arena = std.heap.ArenaAllocator.init(gpa);
     defer analysis_arena.deinit();
 

src/Zcu.zig

@@ -308,8 +308,56 @@ free_type_references: std.ArrayListUnmanaged(u32) = .empty,
 /// Populated by analysis of `AnalUnit.wrap(.{ .memoized_state = s })`, where `s` depends on the element.
 builtin_decl_values: BuiltinDecl.Memoized = .initFill(.none),
 
+incremental_debug_state: if (build_options.enable_debug_extensions) IncrementalDebugState else void =
+    if (build_options.enable_debug_extensions) .init else {},
+
 generation: u32 = 0,
 
+pub const IncrementalDebugState = struct {
+    /// All container types in the ZCU, even dead ones.
+    /// Value is the generation the type was created on.
+    types: std.AutoArrayHashMapUnmanaged(InternPool.Index, u32),
+    /// All `Nav`s in the ZCU, even dead ones.
+    /// Value is the generation the `Nav` was created on.
+    navs: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, u32),
+    /// All `AnalUnit`s in the ZCU, even dead ones.
+    units: std.AutoArrayHashMapUnmanaged(AnalUnit, UnitInfo),
+
+    pub const init: IncrementalDebugState = .{
+        .types = .empty,
+        .navs = .empty,
+        .units = .empty,
+    };
+    pub fn deinit(ids: *IncrementalDebugState, gpa: Allocator) void {
+        for (ids.units.values()) |*unit_info| {
+            unit_info.deps.deinit(gpa);
+        }
+        ids.types.deinit(gpa);
+        ids.navs.deinit(gpa);
+        ids.units.deinit(gpa);
+    }
+
+    pub const UnitInfo = struct {
+        last_update_gen: u32,
+        /// This information isn't easily recoverable from `InternPool`'s dependency storage format.
+        deps: std.ArrayListUnmanaged(InternPool.Dependee),
+    };
+    pub fn getUnitInfo(ids: *IncrementalDebugState, gpa: Allocator, unit: AnalUnit) Allocator.Error!*UnitInfo {
+        const gop = try ids.units.getOrPut(gpa, unit);
+        if (!gop.found_existing) gop.value_ptr.* = .{
+            .last_update_gen = std.math.maxInt(u32),
+            .deps = .empty,
+        };
+        return gop.value_ptr;
+    }
+    pub fn newType(ids: *IncrementalDebugState, zcu: *Zcu, ty: InternPool.Index) Allocator.Error!void {
+        try ids.types.putNoClobber(zcu.gpa, ty, zcu.generation);
+    }
+    pub fn newNav(ids: *IncrementalDebugState, zcu: *Zcu, nav: InternPool.Nav.Index) Allocator.Error!void {
+        try ids.navs.putNoClobber(zcu.gpa, nav, zcu.generation);
+    }
+};
+
 pub const PerThread = @import("Zcu/PerThread.zig");
 
 pub const ImportTableAdapter = struct {
@@ -2746,6 +2794,10 @@ pub fn deinit(zcu: *Zcu) void {
         zcu.free_type_references.deinit(gpa);
 
         if (zcu.resolved_references) |*r| r.deinit(gpa);
+
+        if (zcu.comp.debugIncremental()) {
+            zcu.incremental_debug_state.deinit(gpa);
+        }
     }
     zcu.intern_pool.deinit(gpa);
 }

src/Zcu/PerThread.zig

@@ -635,6 +635,12 @@ pub fn ensureMemoizedStateUpToDate(pt: Zcu.PerThread, stage: InternPool.Memoized
         if (zcu.builtin_decl_values.get(to_check) != .none) return;
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     const any_changed: bool, const new_failed: bool = if (pt.analyzeMemoizedState(stage)) |any_changed|
         .{ any_changed or prev_failed, false }
     else |err| switch (err) {
@@ -784,6 +790,12 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU
         return;
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     const unit_prog_node = zcu.sema_prog_node.start("comptime", 0);
     defer unit_prog_node.end();
 
@@ -958,6 +970,12 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu
         }
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
     defer unit_prog_node.end();
 
@@ -1331,6 +1349,12 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc
         }
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
     defer unit_prog_node.end();
 
@@ -1564,6 +1588,12 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, maybe_coerced_func_index: Inter
         if (func.analysisUnordered(ip).is_analyzed) return;
     }
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     const func_prog_node = zcu.sema_prog_node.start(ip.getNav(func.owner_nav).fqn.toSlice(ip), 0);
     defer func_prog_node.end();
 
@@ -1816,11 +1846,7 @@ fn createFileRootStruct(
     ip.namespacePtr(namespace_index).owner_type = wip_ty.index;
 
     if (zcu.comp.incremental) {
-        try ip.addDependency(
+        try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = tracked_inst });
-            gpa,
-            .wrap(.{ .type = wip_ty.index }),
-            .{ .src_hash = tracked_inst },
-        );
     }
 
     try pt.scanNamespace(namespace_index, decls);
@@ -1832,6 +1858,7 @@ fn createFileRootStruct(
         try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
     }
     zcu.setFileRootType(file_index, wip_ty.index);
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return wip_ty.finish(ip, namespace_index);
 }
 
@@ -2734,10 +2761,11 @@ const ScanDeclIter = struct {
             else => unit: {
                 const name = maybe_name.unwrap().?;
                 const fqn = try namespace.internFullyQualifiedName(ip, gpa, pt.tid, name);
-                const nav = if (existing_unit) |eu|
+                const nav = if (existing_unit) |eu| eu.unwrap().nav_val else nav: {
-                    eu.unwrap().nav_val
+                    const nav = try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace");
-                else
+                    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newNav(zcu, nav);
-                    try ip.createDeclNav(gpa, pt.tid, name, fqn, tracked_inst, namespace_index, decl.kind == .@"usingnamespace");
+                    break :nav nav;
+                };
 
                 const unit: AnalUnit = .wrap(.{ .nav_val = nav });
 
@@ -3911,6 +3939,7 @@ pub fn getExtern(pt: Zcu.PerThread, key: InternPool.Key.Extern) Allocator.Error!
     if (result.new_nav.unwrap()) |nav| {
         // This job depends on any resolve_type_fully jobs queued up before it.
         try pt.zcu.comp.queueJob(.{ .codegen_nav = nav });
+        if (pt.zcu.comp.debugIncremental()) try pt.zcu.incremental_debug_state.newNav(pt.zcu, nav);
     }
     return result.index;
 }
@@ -3979,6 +4008,12 @@ pub fn ensureTypeUpToDate(pt: Zcu.PerThread, ty: InternPool.Index) Zcu.SemaError
     _ = zcu.transitive_failed_analysis.swapRemove(anal_unit);
     zcu.intern_pool.removeDependenciesForDepender(gpa, anal_unit);
 
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, anal_unit);
+        info.last_update_gen = zcu.generation;
+        info.deps.clearRetainingCapacity();
+    }
+
     switch (ip.indexToKey(ty)) {
         .struct_type => return pt.recreateStructType(ty, declared_ty_key),
         .union_type => return pt.recreateUnionType(ty, declared_ty_key),
@@ -4042,11 +4077,7 @@ fn recreateStructType(
     errdefer wip_ty.cancel(ip, pt.tid);
 
     wip_ty.setName(ip, struct_obj.name);
-    try ip.addDependency(
+    try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index });
-        gpa,
-        .wrap(.{ .type = wip_ty.index }),
-        .{ .src_hash = key.zir_index },
-    );
     zcu.namespacePtr(struct_obj.namespace).owner_type = wip_ty.index;
     // No need to re-scan the namespace -- `zirStructDecl` will ultimately do that if the type is still alive.
     try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index });
@@ -4058,6 +4089,7 @@ fn recreateStructType(
         try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
     }
 
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     const new_ty = wip_ty.finish(ip, struct_obj.namespace);
     if (inst_info.inst == .main_struct_inst) {
         // This is the root type of a file! Update the reference.
@@ -4138,11 +4170,7 @@ fn recreateUnionType(
     errdefer wip_ty.cancel(ip, pt.tid);
 
     wip_ty.setName(ip, union_obj.name);
-    try ip.addDependency(
+    try pt.addDependency(.wrap(.{ .type = wip_ty.index }), .{ .src_hash = key.zir_index });
-        gpa,
-        .wrap(.{ .type = wip_ty.index }),
-        .{ .src_hash = key.zir_index },
-    );
     zcu.namespacePtr(namespace_index).owner_type = wip_ty.index;
     // No need to re-scan the namespace -- `zirUnionDecl` will ultimately do that if the type is still alive.
     try zcu.comp.queueJob(.{ .resolve_type_fully = wip_ty.index });
@@ -4154,6 +4182,7 @@ fn recreateUnionType(
         try zcu.comp.queueJob(.{ .codegen_type = wip_ty.index });
     }
 
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     return wip_ty.finish(ip, namespace_index);
 }
 
@@ -4255,6 +4284,7 @@ fn recreateEnumType(
     zcu.namespacePtr(namespace_index).owner_type = wip_ty.index;
     // No need to re-scan the namespace -- `zirEnumDecl` will ultimately do that if the type is still alive.
 
+    if (zcu.comp.debugIncremental()) try zcu.incremental_debug_state.newType(zcu, wip_ty.index);
     wip_ty.prepare(ip, namespace_index);
     done = true;
 
@@ -4432,3 +4462,13 @@ pub fn refValue(pt: Zcu.PerThread, val: InternPool.Index) Zcu.SemaError!InternPo
         .byte_offset = 0,
     } });
 }
+
+pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dependee) Allocator.Error!void {
+    const zcu = pt.zcu;
+    const gpa = zcu.gpa;
+    try zcu.intern_pool.addDependency(gpa, unit, dependee);
+    if (zcu.comp.debugIncremental()) {
+        const info = try zcu.incremental_debug_state.getUnitInfo(gpa, unit);
+        try info.deps.append(gpa, dependee);
+    }
+}

src/main.zig

@@ -677,6 +677,7 @@ const usage_build_generic =
     \\  --debug-compile-errors       Crash with helpful diagnostics at the first compile error
     \\  --debug-link-snapshot        Enable dumping of the linker's state in JSON format
     \\  --debug-rt                   Debug compiler runtime libraries
+    \\  --debug-incremental          Enable incremental compilation debug features
     \\
 ;
 
@@ -832,6 +833,7 @@ fn buildOutputType(
     var data_sections = false;
     var listen: Listen = .none;
     var debug_compile_errors = false;
+    var debug_incremental = false;
     var verbose_link = (native_os != .wasi or builtin.link_libc) and
         EnvVar.ZIG_VERBOSE_LINK.isSet();
     var verbose_cc = (native_os != .wasi or builtin.link_libc) and
@@ -1383,6 +1385,12 @@ fn buildOutputType(
                         }
                     } else if (mem.eql(u8, arg, "--debug-rt")) {
                         debug_compiler_runtime_libs = true;
+                    } else if (mem.eql(u8, arg, "--debug-incremental")) {
+                        if (build_options.enable_debug_extensions) {
+                            debug_incremental = true;
+                        } else {
+                            warn("Zig was compiled without debug extensions. --debug-incremental has no effect.", .{});
+                        }
                     } else if (mem.eql(u8, arg, "-fincremental")) {
                         dev.check(.incremental);
                         opt_incremental = true;
@@ -3460,6 +3468,9 @@ fn buildOutputType(
     };
 
     const incremental = opt_incremental orelse false;
+    if (debug_incremental and !incremental) {
+        fatal("--debug-incremental requires -fincremental", .{});
+    }
 
     const disable_lld_caching = !output_to_cache;
 
@@ -3592,6 +3603,7 @@ fn buildOutputType(
         .cache_mode = cache_mode,
         .subsystem = subsystem,
         .debug_compile_errors = debug_compile_errors,
+        .debug_incremental = debug_incremental,
         .incremental = incremental,
         .enable_link_snapshots = enable_link_snapshots,
         .install_name = install_name,
@@ -4195,9 +4207,25 @@ fn serve(
     const main_progress_node = std.Progress.start(.{});
     const file_system_inputs = comp.file_system_inputs.?;
 
+    const IncrementalDebugServer = if (build_options.enable_debug_extensions)
+        @import("IncrementalDebugServer.zig")
+    else
+        void;
+
+    var ids: IncrementalDebugServer = if (comp.debugIncremental()) ids: {
+        break :ids .init(comp.zcu orelse @panic("--debug-incremental requires a ZCU"));
+    } else undefined;
+    defer if (comp.debugIncremental()) ids.deinit();
+
+    if (comp.debugIncremental()) ids.spawn();
+
     while (true) {
         const hdr = try server.receiveMessage();
 
+        // Lock the debug server while hanling the message.
+        if (comp.debugIncremental()) ids.mutex.lock();
+        defer if (comp.debugIncremental()) ids.mutex.unlock();
+
         switch (hdr.tag) {
             .exit => return cleanExit(),
             .update => {