make zig compiler processes live across rebuilds

Changes the `make` function signature to take an options struct, which
additionally includes `watch: bool`. I intentionally am not exposing
this information to configure phase logic.

Also adds global zig cache to the compiler cache prefixes.

Closes #20600

lib/compiler/build_runner.zig

@@ -1031,7 +1031,11 @@ fn workerMakeOneStep(
     const sub_prog_node = prog_node.start(s.name, 0);
     defer sub_prog_node.end();
 
-    const make_result = s.make(sub_prog_node);
+    const make_result = s.make(.{
+        .progress_node = sub_prog_node,
+        .thread_pool = thread_pool,
+        .watch = run.watch,
+    });
 
     // No matter the result, we want to display error/warning messages.
     const show_compile_errors = !run.prominent_compile_errors and

lib/std/Build.zig

@@ -1078,8 +1078,8 @@ pub fn getUninstallStep(b: *Build) *Step {
     return &b.uninstall_tls.step;
 }
 
-fn makeUninstall(uninstall_step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void {
+    _ = options;
     const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step);
     const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls);
 

lib/std/Build/Step.zig

@@ -68,7 +68,13 @@ pub const TestResults = struct {
     }
 };
 
-pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerror!void;
+pub const MakeOptions = struct {
+    progress_node: std.Progress.Node,
+    thread_pool: *std.Thread.Pool,
+    watch: bool,
+};
+
+pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
 
 pub const State = enum {
     precheck_unstarted,
@@ -219,10 +225,10 @@ pub fn init(options: StepOptions) Step {
 /// If the Step's `make` function reports `error.MakeFailed`, it indicates they
 /// have already reported the error. Otherwise, we add a simple error report
 /// here.
-pub fn make(s: *Step, prog_node: std.Progress.Node) error{ MakeFailed, MakeSkipped }!void {
+pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
     const arena = s.owner.allocator;
 
-    s.makeFn(s, prog_node) catch |err| switch (err) {
+    s.makeFn(s, options) catch |err| switch (err) {
         error.MakeFailed => return error.MakeFailed,
         error.MakeSkipped => return error.MakeSkipped,
         else => {
@@ -260,8 +266,8 @@ pub fn getStackTrace(s: *Step) ?std.builtin.StackTrace {
     };
 }
 
-fn makeNoOp(step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void {
+    _ = options;
 
     var all_cached = true;
 
@@ -352,13 +358,25 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO
     try step.result_error_msgs.append(arena, msg);
 }
 
+pub const ZigProcess = struct {
+    child: std.process.Child,
+    poller: std.io.Poller(StreamEnum),
+
+    pub const StreamEnum = enum { stdout, stderr };
+};
+
 /// Assumes that argv contains `--listen=-` and that the process being spawned
 /// is the zig compiler - the same version that compiled the build runner.
 pub fn evalZigProcess(
     s: *Step,
     argv: []const []const u8,
     prog_node: std.Progress.Node,
+    watch: bool,
 ) !?[]const u8 {
+    if (s.getZigProcess()) |zp| {
+        assert(watch);
+        return zigProcessUpdate(s, zp, watch);
+    }
     assert(argv.len != 0);
     const b = s.owner;
     const arena = b.allocator;
@@ -378,29 +396,76 @@ pub fn evalZigProcess(
     child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{
         argv[0], @errorName(err),
     });
+
+    const zp = try arena.create(ZigProcess);
+    zp.* = .{
+        .child = child,
+        .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{
+            .stdout = child.stdout.?,
+            .stderr = child.stderr.?,
+        }),
+    };
+    if (watch) s.setZigProcess(zp);
+    defer if (!watch) zp.poller.deinit();
+
+    const result = try zigProcessUpdate(s, zp, watch);
+
+    if (!watch) {
+        // Send EOF to stdin.
+        zp.child.stdin.?.close();
+        zp.child.stdin = null;
+
+        const term = zp.child.wait() catch |err| {
+            return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
+        };
+        s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
+
+        // Special handling for Compile step that is expecting compile errors.
+        if (s.cast(Compile)) |compile| switch (term) {
+            .Exited => {
+                // Note that the exit code may be 0 in this case due to the
+                // compiler server protocol.
+                if (compile.expect_errors != null) {
+                    return error.NeedCompileErrorCheck;
+                }
+            },
+            else => {},
+        };
+
+        try handleChildProcessTerm(s, term, null, argv);
+    }
+
+    if (s.result_error_bundle.errorMessageCount() > 0) {
+        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
+            s.result_error_bundle.errorMessageCount(),
+            try allocPrintCmd(arena, null, argv),
+        });
+    }
+
+    return result;
+}
+
+fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
+    const b = s.owner;
+    const arena = b.allocator;
+
     var timer = try std.time.Timer.start();
 
-    var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
-        .stdout = child.stdout.?,
-        .stderr = child.stderr.?,
-    });
-    defer poller.deinit();
-
-    try sendMessage(child.stdin.?, .update);
-    try sendMessage(child.stdin.?, .exit);
+    try sendMessage(zp.child.stdin.?, .update);
+    if (!watch) try sendMessage(zp.child.stdin.?, .exit);
 
     const Header = std.zig.Server.Message.Header;
     var result: ?[]const u8 = null;
 
-    const stdout = poller.fifo(.stdout);
+    const stdout = zp.poller.fifo(.stdout);
 
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const header = stdout.reader().readStruct(Header) catch unreachable;
         while (stdout.readableLength() < header.bytes_len) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const body = stdout.readableSliceOfLen(header.bytes_len);
 
@@ -428,12 +493,22 @@ pub fn evalZigProcess(
                     .string_bytes = try arena.dupe(u8, string_bytes),
                     .extra = extra_array,
                 };
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .emit_bin_path => {
                 const EbpHdr = std.zig.Server.Message.EmitBinPath;
                 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
                 s.result_cached = ebp_hdr.flags.cache_hit;
                 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .file_system_inputs => {
                 s.clearWatchInputs();
@@ -470,6 +545,13 @@ pub fn evalZigProcess(
                             };
                             try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
                         },
+                        .global_cache => {
+                            const path: Build.Cache.Path = .{
+                                .root_dir = s.owner.graph.global_cache_root,
+                                .sub_path = sub_path_dirname,
+                            };
+                            try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
+                        },
                     }
                 }
             },
@@ -479,45 +561,30 @@ pub fn evalZigProcess(
         stdout.discard(body.len);
     }
 
-    const stderr = poller.fifo(.stderr);
+    s.result_duration_ns = timer.read();
+
+    const stderr = zp.poller.fifo(.stderr);
     if (stderr.readableLength() > 0) {
         try s.result_error_msgs.append(arena, try stderr.toOwnedSlice());
     }
 
-    // Send EOF to stdin.
-    child.stdin.?.close();
-    child.stdin = null;
-
-    const term = child.wait() catch |err| {
-        return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
-    };
-    s.result_duration_ns = timer.read();
-    s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
-
-    // Special handling for Compile step that is expecting compile errors.
-    if (s.cast(Compile)) |compile| switch (term) {
-        .Exited => {
-            // Note that the exit code may be 0 in this case due to the
-            // compiler server protocol.
-            if (compile.expect_errors != null) {
-                return error.NeedCompileErrorCheck;
-            }
-        },
-        else => {},
-    };
-
-    try handleChildProcessTerm(s, term, null, argv);
-
-    if (s.result_error_bundle.errorMessageCount() > 0) {
-        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
-            s.result_error_bundle.errorMessageCount(),
-            try allocPrintCmd(arena, null, argv),
-        });
-    }
-
     return result;
 }
 
+fn getZigProcess(s: *Step) ?*ZigProcess {
+    return switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process,
+        else => null,
+    };
+}
+
+fn setZigProcess(s: *Step, zp: *ZigProcess) void {
+    switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process = zp,
+        else => unreachable,
+    }
+}
+
 fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
     const header: std.zig.Client.Message.Header = .{
         .tag = tag,

lib/std/Build/Step/CheckFile.zig

@@ -46,8 +46,8 @@ pub fn setName(check_file: *CheckFile, name: []const u8) void {
     check_file.step.name = name;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const check_file: *CheckFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(check_file.source);

lib/std/Build/Step/CheckObject.zig

@@ -550,8 +550,8 @@ pub fn checkComputeCompare(
     check_object.checks.append(check) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, make_options: Step.MakeOptions) !void {
+    _ = make_options;
     const b = step.owner;
     const gpa = b.allocator;
     const check_object: *CheckObject = @fieldParentPtr("step", step);

lib/std/Build/Step/Compile.zig

@@ -213,6 +213,10 @@ is_linking_libcpp: bool = false,
 
 no_builtin: bool = false,
 
+/// Populated during the make phase when there is a long-lived compiler process.
+/// Managed by the build runner, not user build script.
+zig_process: ?*Step.ZigProcess,
+
 pub const ExpectedCompileErrors = union(enum) {
     contains: []const u8,
     exact: []const []const u8,
@@ -398,6 +402,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
 
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
+
+        .zig_process = null,
     };
 
     compile.root_module.init(owner, options.root_module, compile);
@@ -1735,13 +1741,17 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
     return try zig_args.toOwnedSlice();
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
     const b = step.owner;
     const compile: *Compile = @fieldParentPtr("step", step);
 
     const zig_args = try getZigArgs(compile);
 
-    const maybe_output_bin_path = step.evalZigProcess(zig_args, prog_node) catch |err| switch (err) {
+    const maybe_output_bin_path = step.evalZigProcess(
+        zig_args,
+        options.progress_node,
+        options.watch,
+    ) catch |err| switch (err) {
         error.NeedCompileErrorCheck => {
             assert(compile.expect_errors != null);
             try checkCompileErrors(compile);

lib/std/Build/Step/ConfigHeader.zig

@@ -164,8 +164,8 @@ fn putValue(config_header: *ConfigHeader, field_name: []const u8, comptime T: ty
     }
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const config_header: *ConfigHeader = @fieldParentPtr("step", step);
     if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp);

lib/std/Build/Step/Fail.zig

@@ -24,8 +24,8 @@ pub fn create(owner: *std.Build, error_msg: []const u8) *Fail {
     return fail;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node; // No progress to report.
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options; // No progress to report.
 
     const fail: *Fail = @fieldParentPtr("step", step);
 

lib/std/Build/Step/Fmt.zig

@@ -36,7 +36,9 @@ pub fn create(owner: *std.Build, options: Options) *Fmt {
     return fmt;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
+
     // TODO: if check=false, this means we are modifying source files in place, which
     // is an operation that could race against other operations also modifying source files
     // in place. In this case, this step should obtain a write lock while making those

lib/std/Build/Step/InstallArtifact.zig

@@ -115,8 +115,8 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins
     return install_artifact;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const install_artifact: *InstallArtifact = @fieldParentPtr("step", step);
     const b = step.owner;
     const cwd = fs.cwd();

lib/std/Build/Step/InstallDir.zig

@@ -55,8 +55,8 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir {
     return install_dir;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_dir: *InstallDir = @fieldParentPtr("step", step);
     step.clearWatchInputs();

lib/std/Build/Step/InstallFile.zig

@@ -35,8 +35,8 @@ pub fn create(
     return install_file;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_file: *InstallFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(install_file.source);

lib/std/Build/Step/ObjCopy.zig

@@ -90,7 +90,8 @@ pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath {
     return if (objcopy.output_file_debug) |*file| .{ .generated = .{ .file = file } } else null;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const objcopy: *ObjCopy = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(objcopy.input_file);
@@ -158,7 +159,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
     try argv.appendSlice(&.{ full_src_path, full_dest_path });
 
     try argv.append("--listen=-");
-    _ = try step.evalZigProcess(argv.items, prog_node);
+    _ = try step.evalZigProcess(argv.items, prog_node, false);
 
     objcopy.output_file.path = full_dest_path;
     if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug;

lib/std/Build/Step/Options.zig

@@ -410,9 +410,9 @@ pub fn getOutput(options: *Options) LazyPath {
     return .{ .generated = .{ .file = &options.generated_file } };
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    // This step completes so quickly that no progress is necessary.
-    _ = prog_node;
+fn make(step: *Step, make_options: Step.MakeOptions) !void {
+    // This step completes so quickly that no progress reporting is necessary.
+    _ = make_options;
 
     const b = step.owner;
     const options: *Options = @fieldParentPtr("step", step);

lib/std/Build/Step/RemoveDir.zig

@@ -23,10 +23,8 @@ pub fn create(owner: *std.Build, doomed_path: LazyPath) *RemoveDir {
     return remove_dir;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    // TODO update progress node while walking file system.
-    // Should the standard library support this use case??
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
 
     const b = step.owner;
     const remove_dir: *RemoveDir = @fieldParentPtr("step", step);

lib/std/Build/Step/Run.zig

@@ -595,7 +595,8 @@ const IndexedOutput = struct {
     tag: @typeInfo(Arg).Union.tag_type.?,
     output: *Output,
 };
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const arena = b.allocator;
     const run: *Run = @fieldParentPtr("step", step);

lib/std/Build/Step/TranslateC.zig

@@ -116,7 +116,8 @@ pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) voi
     translate_c.c_macros.append(translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const translate_c: *TranslateC = @fieldParentPtr("step", step);
 
@@ -154,7 +155,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
 
     try argv_list.append(translate_c.source.getPath2(b, step));
 
-    const output_path = try step.evalZigProcess(argv_list.items, prog_node);
+    const output_path = try step.evalZigProcess(argv_list.items, prog_node, false);
 
     translate_c.out_basename = fs.path.basename(output_path.?);
     const output_dir = fs.path.dirname(output_path.?).?;

lib/std/Build/Step/UpdateSourceFiles.zig

@@ -67,8 +67,8 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []
     }) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const usf: *UpdateSourceFiles = @fieldParentPtr("step", step);
 

lib/std/Build/Step/WriteFile.zig

@@ -171,8 +171,8 @@ fn maybeUpdateName(write_file: *WriteFile) void {
     }
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const arena = b.allocator;
     const gpa = arena;

lib/std/zig/Server.zig

@@ -36,6 +36,7 @@ pub const Message = struct {
         cwd,
         zig_lib,
         local_cache,
+        global_cache,
     };
 
     /// Trailing:

src/Compilation.zig

@@ -1363,6 +1363,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
         cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
         cache.addPrefix(options.zig_lib_directory);
         cache.addPrefix(options.local_cache_directory);
+        cache.addPrefix(options.global_cache_directory);
         errdefer cache.manifest_dir.close();
 
         // This is shared hasher state common to zig source and all C source files.
@@ -2358,7 +2359,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
     }
 }
 
-fn appendFileSystemInput(
+pub fn appendFileSystemInput(
     comp: *Compilation,
     file_system_inputs: *std.ArrayListUnmanaged(u8),
     root: Cache.Path,

src/Zcu/PerThread.zig

@@ -1418,6 +1418,10 @@ pub fn importPkg(pt: Zcu.PerThread, mod: *Module) !Zcu.ImportFileResult {
     const sub_file_path = try gpa.dupe(u8, mod.root_src_path);
     errdefer gpa.free(sub_file_path);
 
+    const comp = zcu.comp;
+    if (comp.file_system_inputs) |fsi|
+        try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
+
     const new_file = try gpa.create(Zcu.File);
     errdefer gpa.destroy(new_file);
 
@@ -1527,6 +1531,10 @@ pub fn importFile(
         resolved_root_path, resolved_path, sub_file_path, import_string,
     });
 
+    const comp = zcu.comp;
+    if (comp.file_system_inputs) |fsi|
+        try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
+
     const path_digest = zcu.computePathDigest(mod, sub_file_path);
     const new_file_index = try ip.createFile(gpa, pt.tid, .{
         .bin_digest = path_digest,

test/standalone/cmakedefine/build.zig

@@ -80,8 +80,8 @@ pub fn build(b: *std.Build) void {
     test_step.dependOn(&wrapper_header.step);
 }
 
-fn compare_headers(step: *std.Build.Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn compare_headers(step: *std.Build.Step, options: std.Build.Step.MakeOptions) !void {
+    _ = options;
     const allocator = step.owner.allocator;
     const expected_fmt = "expected_{s}";