add --fuzz CLI argument to zig build

This flag makes the build runner rebuild unit tests after the pipeline
finishes, if it finds any unit tests.

I did not make this integrate with file system watching yet.

The test runner is updated to detect which tests are fuzz tests.

Run step is updated to track which test indexes are fuzz tests.

lib/compiler/build_runner.zig

@@ -10,7 +10,8 @@ const File = std.fs.File;
 const Step = std.Build.Step;
 const Watch = std.Build.Watch;
 const Allocator = std.mem.Allocator;
-const fatal = std.zig.fatal;
+const fatal = std.process.fatal;
+const runner = @This();
 
 pub const root = @import("@build");
 pub const dependencies = @import("@dependencies");
@@ -102,6 +103,7 @@ pub fn main() !void {
     var steps_menu = false;
     var output_tmp_nonce: ?[16]u8 = null;
     var watch = false;
+    var fuzz = false;
     var debounce_interval_ms: u16 = 50;
 
     while (nextArg(args, &arg_idx)) |arg| {
@@ -234,6 +236,8 @@ pub fn main() !void {
                 prominent_compile_errors = true;
             } else if (mem.eql(u8, arg, "--watch")) {
                 watch = true;
+            } else if (mem.eql(u8, arg, "--fuzz")) {
+                fuzz = true;
             } else if (mem.eql(u8, arg, "-fincremental")) {
                 graph.incremental = true;
             } else if (mem.eql(u8, arg, "-fno-incremental")) {
@@ -353,6 +357,7 @@ pub fn main() !void {
         .max_rss_mutex = .{},
         .skip_oom_steps = skip_oom_steps,
         .watch = watch,
+        .fuzz = fuzz,
         .memory_blocked_steps = std.ArrayList(*Step).init(arena),
         .step_stack = .{},
         .prominent_compile_errors = prominent_compile_errors,
@@ -394,6 +399,10 @@ pub fn main() !void {
             },
             else => return err,
         };
+        if (fuzz) {
+            startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node);
+        }
+
         if (!watch) return cleanExit();
 
         switch (builtin.os.tag) {
@@ -430,6 +439,43 @@ pub fn main() !void {
     }
 }
 
+fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
+    {
+        const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
+        defer rebuild_node.end();
+        var count: usize = 0;
+        var wait_group: std.Thread.WaitGroup = .{};
+        defer wait_group.wait();
+        for (all_steps) |step| {
+            const run = step.cast(Step.Run) orelse continue;
+            if (run.fuzz_tests.items.len > 0 and run.producer != null) {
+                thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
+                count += 1;
+            }
+        }
+        if (count == 0) {
+            std.debug.lockStdErr();
+            std.debug.print("no fuzz tests found\n", .{});
+            process.exit(2);
+        }
+        rebuild_node.setEstimatedTotalItems(count);
+    }
+    @panic("TODO do something with the rebuilt unit tests");
+}
+
+fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
+    const compile_step = run.producer.?;
+    const prog_node = parent_prog_node.start(compile_step.step.name, 0);
+    defer prog_node.end();
+    const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
+        std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
+            compile_step.step.name, @errorName(err),
+        });
+        return;
+    };
+    std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path});
+}
+
 fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
     for (all_steps) |step| switch (step.state) {
         .dependency_failure, .failure, .skipped => step.recursiveReset(gpa),
@@ -457,6 +503,7 @@ const Run = struct {
     max_rss_mutex: std.Thread.Mutex,
     skip_oom_steps: bool,
     watch: bool,
+    fuzz: bool,
     memory_blocked_steps: std.ArrayList(*Step),
     step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
     prominent_compile_errors: bool,
@@ -466,6 +513,11 @@ const Run = struct {
     summary: Summary,
     ttyconf: std.io.tty.Config,
     stderr: File,
+
+    fn cleanExit(run: Run) void {
+        if (run.watch or run.fuzz) return;
+        return runner.cleanExit();
+    }
 };
 
 fn prepare(
@@ -614,8 +666,7 @@ fn runStepNames(
         else => false,
     };
     if (failure_count == 0 and failures_only) {
-        if (!run.watch) cleanExit();
-        return;
+        return run.cleanExit();
     }
 
     const ttyconf = run.ttyconf;
@@ -672,8 +723,7 @@ fn runStepNames(
     }
 
     if (failure_count == 0) {
-        if (!run.watch) cleanExit();
-        return;
+        return run.cleanExit();
     }
 
     // Finally, render compile errors at the bottom of the terminal.
@@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\  --skip-oom-steps             Instead of failing, skip steps that would exceed --maxrss
         \\  --fetch                      Exit after fetching dependency tree
         \\  --watch                      Continuously rebuild when source files are modified
+        \\  --fuzz                       Continuously search for unit test failures
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
         \\     -fincremental             Enable incremental compilation
         \\  -fno-incremental             Disable incremental compilation

lib/compiler/test_runner.zig

@@ -143,6 +143,7 @@ fn mainTerminal() void {
     var ok_count: usize = 0;
     var skip_count: usize = 0;
     var fail_count: usize = 0;
+    var fuzz_count: usize = 0;
     const root_node = std.Progress.start(.{
         .root_name = "Test",
         .estimated_total_items = test_fn_list.len,
@@ -168,7 +169,7 @@ fn mainTerminal() void {
         if (!have_tty) {
             std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
         }
-        // Track in a global variable so that `fuzzInput` can see it.
+        is_fuzz_test = false;
         if (test_fn.func()) |_| {
             ok_count += 1;
             test_node.end();
@@ -198,6 +199,7 @@ fn mainTerminal() void {
                 test_node.end();
             },
         }
+        fuzz_count += @intFromBool(is_fuzz_test);
     }
     root_node.end();
     if (ok_count == test_fn_list.len) {
@@ -211,6 +213,9 @@ fn mainTerminal() void {
     if (leaks != 0) {
         std.debug.print("{d} tests leaked memory.\n", .{leaks});
     }
+    if (fuzz_count != 0) {
+        std.debug.print("{d} fuzz tests found.\n", .{fuzz_count});
+    }
     if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
         std.process.exit(1);
     }

lib/std/Build.zig

@@ -977,6 +977,7 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run {
     // Consider that this is declarative; the run step may not be run unless a user
     // option is supplied.
     const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name}));
+    run_step.producer = exe;
     if (exe.kind == .@"test") {
         if (exe.exec_cmd_args) |exec_cmd_args| {
             for (exec_cmd_args) |cmd_arg| {

lib/std/Build/Step/Compile.zig

@@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking
     return path;
 }
 
-fn getZigArgs(compile: *Compile) ![][]const u8 {
+fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
     const step = &compile.step;
     const b = step.owner;
     const arena = b.allocator;
@@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
         try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size}));
     }
 
+    if (fuzz) {
+        try zig_args.append("-ffuzz");
+    }
+
     {
         // Stores system libraries that have already been seen for at least one
         // module, along with any arguments that need to be passed to the
@@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     const b = step.owner;
     const compile: *Compile = @fieldParentPtr("step", step);
 
-    const zig_args = try getZigArgs(compile);
+    const zig_args = try getZigArgs(compile, false);
 
     const maybe_output_bin_path = step.evalZigProcess(
         zig_args,
@@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     }
 }
 
+pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
+    const zig_args = try getZigArgs(c, true);
+    const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false);
+    return maybe_output_bin_path.?;
+}
+
 pub fn doAtomicSymLinks(
     step: *Step,
     output_path: []const u8,
@@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks(
     };
 }
 
-fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
-    const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
-    const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
-    var list = ArrayList(PkgConfigPkg).init(compile.allocator);
+fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg {
+    const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config";
+    const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore);
+    var list = ArrayList(PkgConfigPkg).init(b.allocator);
     errdefer list.deinit();
     var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
     while (line_it.next()) |line| {
@@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE
     return list.toOwnedSlice();
 }
 
-fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
-    if (compile.pkg_config_pkg_list) |res| {
+fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg {
+    if (b.pkg_config_pkg_list) |res| {
         return res;
     }
     var code: u8 = undefined;
-    if (execPkgConfigList(compile, &code)) |list| {
-        compile.pkg_config_pkg_list = list;
+    if (execPkgConfigList(b, &code)) |list| {
+        b.pkg_config_pkg_list = list;
         return list;
     } else |err| {
         const result = switch (err) {
@@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg {
             error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput,
             else => return err,
         };
-        compile.pkg_config_pkg_list = result;
+        b.pkg_config_pkg_list = result;
         return result;
     }
 }

lib/std/Build/Step/Run.zig

@@ -86,6 +86,13 @@ dep_output_file: ?*Output,
 
 has_side_effects: bool,
 
+/// If this is a Zig unit test binary, this tracks the indexes of the unit
+/// tests that are also fuzz tests.
+fuzz_tests: std.ArrayListUnmanaged(u32),
+
+/// If this Run step was produced by a Compile step, it is tracked here.
+producer: ?*Step.Compile,
+
 pub const StdIn = union(enum) {
     none,
     bytes: []const u8,
@@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run {
         .captured_stderr = null,
         .dep_output_file = null,
         .has_side_effects = false,
+        .fuzz_tests = .{},
+        .producer = null,
     };
     return run;
 }
@@ -1347,6 +1356,8 @@ fn evalZigTest(
     var sub_prog_node: ?std.Progress.Node = null;
     defer if (sub_prog_node) |n| n.end();
 
+    run.fuzz_tests.clearRetainingCapacity();
+
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
             if (!(try poller.poll())) break :poll;
@@ -1404,6 +1415,8 @@ fn evalZigTest(
                 leak_count +|= @intFromBool(tr_hdr.flags.leak);
                 log_err_count +|= tr_hdr.flags.log_err_count;
 
+                if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
+
                 if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) {
                     const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0);
                     const orig_msg = stderr.readableSlice(0);