From bce3b1efb0879ba2f0da4d215c3190f3e8a4345b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 23 Jul 2024 22:59:25 -0700 Subject: [PATCH] build runner sends a start_fuzzing message to test runner --- lib/compiler/build_runner.zig | 16 +++--- lib/std/Build/Fuzz.zig | 84 +++++++++++++++++++++++++----- lib/std/Build/Step/Run.zig | 96 ++++++++++++++++++++++++++++++----- lib/std/zig/Client.zig | 3 ++ 4 files changed, 167 insertions(+), 32 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index e27b4e8762..8c22e6d291 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -401,7 +401,7 @@ pub fn main() !void { else => return err, }; if (fuzz) { - Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node); + Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node); } if (!watch) return cleanExit(); @@ -1072,7 +1072,7 @@ fn workerMakeOneStep( std.debug.lockStdErr(); defer std.debug.unlockStdErr(); - printErrorMessages(b, s, run) catch {}; + printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; } handle_result: { @@ -1125,10 +1125,14 @@ fn workerMakeOneStep( } } -fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void { +pub fn printErrorMessages( + b: *std.Build, + failing_step: *Step, + ttyconf: std.io.tty.Config, + stderr: File, + prominent_compile_errors: bool, +) !void { const gpa = b.allocator; - const stderr = run.stderr; - const ttyconf = run.ttyconf; // Provide context for where these error messages are coming from by // printing the corresponding Step subtree. @@ -1166,7 +1170,7 @@ fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void } } - if (!run.prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) + if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0) try failing_step.result_error_bundle.renderToWriter(renderOptions(ttyconf), stderr.writer()); for (failing_step.result_error_msgs.items) |msg| { diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index a0e754fab1..458d7e5d8b 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -3,9 +3,15 @@ const Fuzz = @This(); const Step = std.Build.Step; const assert = std.debug.assert; const fatal = std.process.fatal; +const build_runner = @import("root"); -pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { - { +pub fn start( + thread_pool: *std.Thread.Pool, + all_steps: []const *Step, + ttyconf: std.io.tty.Config, + prog_node: std.Progress.Node, +) void { + const count = block: { const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); var count: usize = 0; @@ -14,13 +20,14 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: for (all_steps) |step| { const run = step.cast(Step.Run) orelse continue; if (run.fuzz_tests.items.len > 0 and run.producer != null) { - thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node }); count += 1; } } if (count == 0) fatal("no fuzz tests found", .{}); rebuild_node.setEstimatedTotalItems(count); - } + break :block count; + }; // Detect failure. for (all_steps) |step| { @@ -29,18 +36,69 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } - @panic("TODO do something with the rebuilt unit tests"); + { + const rebuild_node = prog_node.start("Fuzzing", count); + defer rebuild_node.end(); + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + for (run.fuzz_tests.items) |unit_test_index| { + assert(run.rebuilt_executable != null); + thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node }); + } + } + } + + fatal("all fuzz workers crashed", .{}); } -fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { +fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { const compile_step = run.producer.?; const prog_node = parent_prog_node.start(compile_step.step.name, 0); defer prog_node.end(); - const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { - std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ - compile_step.step.name, @errorName(err), - }); - return; - }; - run.rebuilt_executable = rebuilt_bin_path; + if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| { + run.rebuilt_executable = rebuilt_bin_path; + } else |err| switch (err) { + error.MakeFailed => { + const b = run.step.owner; + const stderr = std.io.getStdErr(); + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {}; + }, + else => { + std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{ + compile_step.step.name, @errorName(err), + }); + }, + } +} + +fn fuzzWorkerRun( + run: *Step.Run, + unit_test_index: u32, + ttyconf: std.io.tty.Config, + parent_prog_node: std.Progress.Node, +) void { + const test_name = run.cached_test_metadata.?.testName(unit_test_index); + + const prog_node = parent_prog_node.start(test_name, 0); + defer prog_node.end(); + + run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) { + error.MakeFailed => { + const b = run.step.owner; + const stderr = std.io.getStdErr(); + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {}; + }, + else => { + std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{ + run.step.name, test_name, @errorName(err), + }); + }, + }; } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 9b7997306f..43e3ddedfd 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -89,6 +89,8 @@ has_side_effects: bool, /// If this is a Zig unit test binary, this tracks the indexes of the unit /// tests that are also fuzz tests. fuzz_tests: std.ArrayListUnmanaged(u32), +cached_test_metadata: ?CachedTestMetadata = null, + /// Populated during the fuzz phase if this run step corresponds to a unit test /// executable that contains fuzz tests. rebuilt_executable: ?[]const u8, @@ -754,7 +756,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path }); } - try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node); + try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null); if (!has_side_effects) try step.writeManifestAndWatch(&man); return; }; @@ -784,7 +786,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path }); } - try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null); const dep_file_dir = std.fs.cwd(); const dep_file_basename = dep_output_file.generated_file.getPath(); @@ -843,6 +845,38 @@ fn make(step: *Step, options: Step.MakeOptions) !void { ); } +pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void { + const step = &run.step; + const b = step.owner; + const arena = b.allocator; + var argv_list: std.ArrayListUnmanaged([]const u8) = .{}; + for (run.argv.items) |arg| { + switch (arg) { + .bytes => |bytes| { + try argv_list.append(arena, bytes); + }, + .lazy_path => |file| { + const file_path = file.lazy_path.getPath2(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path })); + }, + .directory_source => |file| { + const file_path = file.lazy_path.getPath2(b, step); + try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path })); + }, + .artifact => |pa| { + const artifact = pa.artifact; + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; + try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path })); + }, + .output_file, .output_directory => unreachable, + } + } + const has_side_effects = false; + const rand_int = std.crypto.random.int(u64); + const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index); +} + fn populateGeneratedPaths( arena: std.mem.Allocator, output_placeholders: []const IndexedOutput, @@ -921,6 +955,7 @@ fn runCommand( has_side_effects: bool, output_dir_path: []const u8, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !void { const step = &run.step; const b = step.owner; @@ -939,7 +974,7 @@ fn runCommand( var interp_argv = std.ArrayList([]const u8).init(b.allocator); defer interp_argv.deinit(); - const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node) catch |err| term: { + const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1075,7 +1110,7 @@ fn runCommand( try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items); - break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node) catch |e| { + break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; return step.fail("unable to spawn interpreter {s}: {s}", .{ @@ -1090,6 +1125,15 @@ fn runCommand( step.result_duration_ns = result.elapsed_ns; step.result_peak_rss = result.peak_rss; step.test_results = result.stdio.test_results; + if (result.stdio.test_metadata) |tm| + run.cached_test_metadata = tm.toCachedTestMetadata(); + + const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; + + if (fuzz_unit_test_index != null) { + try step.handleChildProcessTerm(result.term, cwd, final_argv); + return; + } // Capture stdout and stderr to GeneratedFile objects. const Stream = struct { @@ -1126,8 +1170,6 @@ fn runCommand( } } - const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; - switch (run.stdio) { .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { @@ -1253,10 +1295,16 @@ fn spawnChildAndCollect( argv: []const []const u8, has_side_effects: bool, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !ChildProcResult { const b = run.step.owner; const arena = b.allocator; + if (fuzz_unit_test_index != null) { + assert(!has_side_effects); + assert(run.stdio == .zig_test); + } + var child = std.process.Child.init(argv, arena); if (run.cwd) |lazy_cwd| { child.cwd = lazy_cwd.getPath2(b, &run.step); @@ -1306,7 +1354,7 @@ fn spawnChildAndCollect( var timer = try std.time.Timer.start(); const result = if (run.stdio == .zig_test) - evalZigTest(run, &child, prog_node) + evalZigTest(run, &child, prog_node, fuzz_unit_test_index) else evalGeneric(run, &child); @@ -1332,6 +1380,7 @@ fn evalZigTest( run: *Run, child: *std.process.Child, prog_node: std.Progress.Node, + fuzz_unit_test_index: ?u32, ) !StdIoResult { const gpa = run.step.owner.allocator; const arena = run.step.owner.allocator; @@ -1342,7 +1391,12 @@ fn evalZigTest( }); defer poller.deinit(); - try sendMessage(child.stdin.?, .query_test_metadata); + if (fuzz_unit_test_index) |index| { + try sendRunTestMessage(child.stdin.?, .start_fuzzing, index); + } else { + run.fuzz_tests.clearRetainingCapacity(); + try sendMessage(child.stdin.?, .query_test_metadata); + } const Header = std.zig.Server.Message.Header; @@ -1360,8 +1414,6 @@ fn evalZigTest( var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); - run.fuzz_tests.clearRetainingCapacity(); - poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { if (!(try poller.poll())) break :poll; @@ -1382,6 +1434,7 @@ fn evalZigTest( } }, .test_metadata => { + assert(fuzz_unit_test_index == null); const TmHdr = std.zig.Server.Message.TestMetadata; const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body)); test_count = tm_hdr.tests_len; @@ -1410,6 +1463,7 @@ fn evalZigTest( try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); }, .test_results => { + assert(fuzz_unit_test_index == null); const md = metadata.?; const TrHdr = std.zig.Server.Message.TestResults; @@ -1479,7 +1533,23 @@ const TestMetadata = struct { next_index: u32, prog_node: std.Progress.Node, + fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata { + return .{ + .names = tm.names, + .string_bytes = tm.string_bytes, + }; + } + fn testName(tm: TestMetadata, index: u32) []const u8 { + return tm.toCachedTestMetadata().testName(index); + } +}; + +pub const CachedTestMetadata = struct { + names: []const u32, + string_bytes: []const u8, + + pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 { return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); } }; @@ -1495,7 +1565,7 @@ fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Pr if (sub_prog_node.*) |n| n.end(); sub_prog_node.* = metadata.prog_node.start(name, 0); - try sendRunTestMessage(in, i); + try sendRunTestMessage(in, .run_test, i); return; } else { try sendMessage(in, .exit); @@ -1510,9 +1580,9 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { try file.writeAll(std.mem.asBytes(&header)); } -fn sendRunTestMessage(file: std.fs.File, index: u32) !void { +fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void { const header: std.zig.Client.Message.Header = .{ - .tag = .run_test, + .tag = tag, .bytes_len = 4, }; const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index); diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig index af4c29d37d..345b9f9797 100644 --- a/lib/std/zig/Client.zig +++ b/lib/std/zig/Client.zig @@ -33,6 +33,9 @@ pub const Message = struct { /// Ask the test runner to run a particular test. /// The message body is a u32 test index. run_test, + /// Ask the test runner to start fuzzing a particular test. + /// The message body is a u32 test index. + start_fuzzing, _, };