Merge pull request #25342 from ziglang/fuzz-limit

fuzzing: implement limited fuzzing
This commit is contained in:
Andrew Kelley 2025-09-26 05:28:46 -07:00 committed by GitHub
commit e0dc2e4e3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 443 additions and 125 deletions

View file

@ -112,7 +112,7 @@ pub fn main() !void {
var steps_menu = false;
var output_tmp_nonce: ?[16]u8 = null;
var watch = false;
var fuzz = false;
var fuzz: ?std.Build.Fuzz.Mode = null;
var debounce_interval_ms: u16 = 50;
var webui_listen: ?std.net.Address = null;
@ -274,10 +274,44 @@ pub fn main() !void {
webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
}
} else if (mem.eql(u8, arg, "--fuzz")) {
fuzz = true;
fuzz = .{ .forever = undefined };
if (webui_listen == null) {
webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
}
} else if (mem.startsWith(u8, arg, "--fuzz=")) {
const value = arg["--fuzz=".len..];
if (value.len == 0) fatal("missing argument to --fuzz", .{});
const unit: u8 = value[value.len - 1];
const digits = switch (unit) {
'0'...'9' => value,
'K', 'M', 'G' => value[0 .. value.len - 1],
else => fatal(
"invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
.{},
),
};
const amount = std.fmt.parseInt(u64, digits, 10) catch {
fatal(
"invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
.{},
);
};
const normalized_amount = std.math.mul(u64, amount, switch (unit) {
else => unreachable,
'0'...'9' => 1,
'K' => 1000,
'M' => 1_000_000,
'G' => 1_000_000_000,
}) catch fatal("fuzzing limit amount overflows u64", .{});
fuzz = .{
.limit = .{
.amount = normalized_amount,
},
};
} else if (mem.eql(u8, arg, "-fincremental")) {
graph.incremental = true;
} else if (mem.eql(u8, arg, "-fno-incremental")) {
@ -476,6 +510,7 @@ pub fn main() !void {
targets.items,
main_progress_node,
&run,
fuzz,
) catch |err| switch (err) {
error.UncleanExit => {
assert(!run.watch and run.web_server == null);
@ -485,7 +520,12 @@ pub fn main() !void {
};
if (run.web_server) |*web_server| {
web_server.finishBuild(.{ .fuzz = fuzz });
if (fuzz) |mode| if (mode != .forever) fatal(
"error: limited fuzzing is not implemented yet for --webui",
.{},
);
web_server.finishBuild(.{ .fuzz = fuzz != null });
}
if (!watch and run.web_server == null) {
@ -651,6 +691,7 @@ fn runStepNames(
step_names: []const []const u8,
parent_prog_node: std.Progress.Node,
run: *Run,
fuzz: ?std.Build.Fuzz.Mode,
) !void {
const gpa = run.gpa;
const step_stack = &run.step_stack;
@ -676,6 +717,7 @@ fn runStepNames(
});
}
}
assert(run.memory_blocked_steps.items.len == 0);
var test_skip_count: usize = 0;
@ -724,6 +766,45 @@ fn runStepNames(
}
}
const ttyconf = run.ttyconf;
if (fuzz) |mode| blk: {
switch (builtin.os.tag) {
// Current implementation depends on two things that need to be ported to Windows:
// * Memory-mapping to share data between the fuzzer and build runner.
// * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
// many addresses to source locations).
.windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
else => {},
}
if (@bitSizeOf(usize) != 64) {
// Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
// being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
// on 32-bit platforms.
// Affects or affected by issues #5185, #22523, and #22464.
fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
}
switch (mode) {
.forever => break :blk,
.limit => {},
}
assert(mode == .limit);
var f = std.Build.Fuzz.init(
gpa,
thread_pool,
step_stack.keys(),
parent_prog_node,
ttyconf,
mode,
) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
defer f.deinit();
f.start();
f.waitAndPrintReport();
}
// A proper command line application defaults to silently succeeding.
// The user may request verbose mode if they have a different preference.
const failures_only = switch (run.summary) {
@ -737,8 +818,6 @@ fn runStepNames(
std.Progress.setStatus(.failure);
}
const ttyconf = run.ttyconf;
if (run.summary != .none) {
const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
defer std.debug.unlockStderrWriter();
@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
\\ --watch Continuously rebuild when source files are modified
\\ --debounce <ms> Delay before rebuilding after changed file detected
\\ --webui[=ip] Enable the web interface on the given IP address
\\ --fuzz Continuously search for unit test failures (implies '--webui')
\\ --fuzz[=limit] Continuously search for unit test failures with an optional
\\ limit to the max number of iterations. The argument supports
\\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
\\ '--webui' when no limit is specified.
\\ --time-report Force full rebuild and provide detailed information on
\\ compilation time of Zig source code (implies '--webui')
\\ -fincremental Enable incremental compilation

View file

@ -2,6 +2,7 @@
const builtin = @import("builtin");
const std = @import("std");
const fatal = std.process.fatal;
const testing = std.testing;
const assert = std.debug.assert;
const fuzz_abi = std.Build.abi.fuzz;
@ -55,12 +56,13 @@ pub fn main() void {
}
}
fba.reset();
if (builtin.fuzz) {
const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
}
fba.reset();
if (listen) {
return mainServer() catch @panic("internal test runner failure");
} else {
@ -79,8 +81,13 @@ fn mainServer() !void {
});
if (builtin.fuzz) {
const coverage_id = fuzz_abi.fuzzer_coverage_id();
try server.serveU64Message(.coverage_id, coverage_id);
const coverage = fuzz_abi.fuzzer_coverage();
try server.serveCoverageIdMessage(
coverage.id,
coverage.runs,
coverage.unique,
coverage.seen,
);
}
while (true) {
@ -158,6 +165,9 @@ fn mainServer() !void {
if (!builtin.fuzz) unreachable;
const index = try server.receiveBody_u32();
const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
const amount_or_instance = try server.receiveBody_u64();
const test_fn = builtin.test_functions[index];
const entry_addr = @intFromPtr(test_fn.func);
@ -165,6 +175,8 @@ fn mainServer() !void {
defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
is_fuzz_test = false;
fuzz_test_index = index;
fuzz_mode = mode;
fuzz_amount_or_instance = amount_or_instance;
test_fn.func() catch |err| switch (err) {
error.SkipZigTest => return,
@ -172,12 +184,14 @@ fn mainServer() !void {
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.debug.print("failed with error.{t}\n", .{err});
std.process.exit(1);
},
};
if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
if (log_err_count != 0) @panic("error logs detected");
assert(mode != .forever);
std.process.exit(0);
},
else => {
@ -240,11 +254,11 @@ fn mainTerminal() void {
else => {
fail_count += 1;
if (have_tty) {
std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{
i + 1, test_fn_list.len, test_fn.name, @errorName(err),
std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
i + 1, test_fn_list.len, test_fn.name, err,
});
} else {
std.debug.print("FAIL ({s})\n", .{@errorName(err)});
std.debug.print("FAIL ({t})\n", .{err});
}
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {
var is_fuzz_test: bool = undefined;
var fuzz_test_index: u32 = undefined;
var fuzz_mode: fuzz_abi.LimitKind = undefined;
var fuzz_amount_or_instance: u64 = undefined;
pub fn fuzz(
context: anytype,
@ -383,7 +399,7 @@ pub fn fuzz(
else => {
std.debug.lockStdErr();
if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.debug.print("failed with error.{t}\n", .{err});
std.process.exit(1);
},
};
@ -401,9 +417,11 @@ pub fn fuzz(
global.ctx = context;
fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
for (options.corpus) |elem|
fuzz_abi.fuzzer_new_input(.fromSlice(elem));
fuzz_abi.fuzzer_main();
fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
return;
}

View file

@ -1,5 +1,6 @@
const builtin = @import("builtin");
const std = @import("std");
const fatal = std.process.fatal;
const mem = std.mem;
const math = std.math;
const Allocator = mem.Allocator;
@ -105,6 +106,7 @@ const Executable = struct {
const coverage_file_len = @sizeOf(abi.SeenPcsHeader) +
pc_bitset_usizes * @sizeOf(usize) +
pcs.len * @sizeOf(usize);
if (populate) {
defer coverage_file.lock(.shared) catch |e| panic(
"failed to demote lock for coverage file '{s}': {t}",
@ -510,7 +512,7 @@ const Fuzzer = struct {
self.corpus_pos = 0;
const rng = self.rng.random();
while (true) {
const m = while (true) {
const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)];
if (!m.mutate(
rng,
@ -522,53 +524,53 @@ const Fuzzer = struct {
inst.const_vals8.items,
inst.const_vals16.items,
)) continue;
break m;
};
self.run();
if (inst.isFresh()) {
@branchHint(.unlikely);
self.run();
const header = mem.bytesAsValue(
abi.SeenPcsHeader,
exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
);
_ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
if (inst.isFresh()) {
@branchHint(.unlikely);
inst.setFresh();
self.minimizeInput();
inst.updateSeen();
const header = mem.bytesAsValue(
abi.SeenPcsHeader,
exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
);
_ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
// An empty-input has always been tried, so if an empty input is fresh then the
// test has to be non-deterministic. This has to be checked as duplicate empty
// entries are not allowed.
if (self.input.items.len - 8 == 0) {
std.log.warn("non-deterministic test (empty input produces different hits)", .{});
_ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
return;
}
inst.setFresh();
self.minimizeInput();
inst.updateSeen();
const arena = self.arena_ctx.allocator();
const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
self.corpus.append(gpa, bytes) catch @panic("OOM");
self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
// Write new corpus to cache
var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
self.corpus_dir.writeFile(.{
.sub_path = std.fmt.bufPrint(
&name_buf,
"{x}",
.{self.corpus_dir_idx},
) catch unreachable,
.data = bytes,
}) catch |e| panic(
"failed to write corpus file '{x}': {t}",
.{ self.corpus_dir_idx, e },
);
self.corpus_dir_idx += 1;
// An empty-input has always been tried, so if an empty input is fresh then the
// test has to be non-deterministic. This has to be checked as duplicate empty
// entries are not allowed.
if (self.input.items.len - 8 == 0) {
std.log.warn("non-deterministic test (empty input produces different hits)", .{});
_ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
return;
}
break;
const arena = self.arena_ctx.allocator();
const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
self.corpus.append(gpa, bytes) catch @panic("OOM");
self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
// Write new corpus to cache
var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
self.corpus_dir.writeFile(.{
.sub_path = std.fmt.bufPrint(
&name_buf,
"{x}",
.{self.corpus_dir_idx},
) catch unreachable,
.data = bytes,
}) catch |e| panic(
"failed to write corpus file '{x}': {t}",
.{ self.corpus_dir_idx, e },
);
self.corpus_dir_idx += 1;
}
}
};
@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void {
}
/// Invalid until `fuzzer_init` is called.
export fn fuzzer_coverage_id() u64 {
return exec.pc_digest;
export fn fuzzer_coverage() abi.Coverage {
const coverage_id = exec.pc_digest;
const header: *const abi.SeenPcsHeader = @ptrCast(@volatileCast(exec.shared_seen_pcs.items.ptr));
var seen_count: usize = 0;
for (header.seenBits()) |chunk| {
seen_count += @popCount(chunk);
}
return .{
.id = coverage_id,
.runs = header.n_runs,
.unique = header.unique_runs,
.seen = seen_count,
};
}
/// fuzzer_init must be called beforehand
@ -600,9 +615,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {
}
/// fuzzer_init_test must be called first
export fn fuzzer_main() void {
while (true) {
fuzzer.cycle();
export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
switch (limit_kind) {
.forever => while (true) fuzzer.cycle(),
.iterations => for (0..amount) |_| fuzzer.cycle(),
}
}

View file

@ -8,17 +8,22 @@ const Allocator = std.mem.Allocator;
const log = std.log;
const Coverage = std.debug.Coverage;
const abi = Build.abi.fuzz;
const tty = std.Io.tty;
const Fuzz = @This();
const build_runner = @import("root");
ws: *Build.WebServer,
gpa: Allocator,
mode: Mode,
/// Allocated into `ws.gpa`.
/// Allocated into `gpa`.
run_steps: []const *Step.Run,
wait_group: std.Thread.WaitGroup,
root_prog_node: std.Progress.Node,
prog_node: std.Progress.Node,
thread_pool: *std.Thread.Pool,
ttyconf: tty.Config,
/// Protects `coverage_files`.
coverage_mutex: std.Thread.Mutex,
@ -28,9 +33,23 @@ queue_mutex: std.Thread.Mutex,
queue_cond: std.Thread.Condition,
msg_queue: std.ArrayListUnmanaged(Msg),
pub const Mode = union(enum) {
forever: struct { ws: *Build.WebServer },
limit: Limited,
pub const Limited = struct {
amount: u64,
};
};
const Msg = union(enum) {
coverage: struct {
id: u64,
cumulative: struct {
runs: u64,
unique: u64,
coverage: u64,
},
run: *Step.Run,
},
entry_point: struct {
@ -54,23 +73,28 @@ const CoverageMap = struct {
}
};
pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
const gpa = ws.gpa;
pub fn init(
gpa: Allocator,
thread_pool: *std.Thread.Pool,
all_steps: []const *Build.Step,
root_prog_node: std.Progress.Node,
ttyconf: tty.Config,
mode: Mode,
) Allocator.Error!Fuzz {
const run_steps: []const *Step.Run = steps: {
var steps: std.ArrayListUnmanaged(*Step.Run) = .empty;
defer steps.deinit(gpa);
const rebuild_node = ws.root_prog_node.start("Rebuilding Unit Tests", 0);
const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
defer rebuild_node.end();
var rebuild_wg: std.Thread.WaitGroup = .{};
defer rebuild_wg.wait();
for (ws.all_steps) |step| {
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
if (run.producer == null) continue;
if (run.fuzz_tests.items.len == 0) continue;
try steps.append(gpa, run);
ws.thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ws.ttyconf, rebuild_node });
thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
}
if (steps.items.len == 0) fatal("no fuzz tests found", .{});
@ -86,9 +110,13 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
}
return .{
.ws = ws,
.gpa = gpa,
.mode = mode,
.run_steps = run_steps,
.wait_group = .{},
.thread_pool = thread_pool,
.ttyconf = ttyconf,
.root_prog_node = root_prog_node,
.prog_node = .none,
.coverage_files = .empty,
.coverage_mutex = .{},
@ -99,32 +127,31 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
}
pub fn start(fuzz: *Fuzz) void {
const ws = fuzz.ws;
fuzz.prog_node = ws.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
// For polling messages and sending updates to subscribers.
fuzz.wait_group.start();
_ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
fuzz.wait_group.finish();
fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
};
if (fuzz.mode == .forever) {
// For polling messages and sending updates to subscribers.
fuzz.wait_group.start();
_ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
fuzz.wait_group.finish();
fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
};
}
for (fuzz.run_steps) |run| {
for (run.fuzz_tests.items) |unit_test_index| {
assert(run.rebuilt_executable != null);
ws.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
fuzz, run, unit_test_index,
});
}
}
}
pub fn deinit(fuzz: *Fuzz) void {
if (true) @panic("TODO: terminate the fuzzer processes");
fuzz.wait_group.wait();
fuzz.prog_node.end();
const gpa = fuzz.ws.gpa;
gpa.free(fuzz.run_steps);
pub fn deinit(fuzz: *Fuzz) void {
if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
fuzz.prog_node.end();
fuzz.gpa.free(fuzz.run_steps);
}
fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.Io.tty.Config, parent_prog_node: std.Progress.Node) void {
@ -177,7 +204,7 @@ fn fuzzWorkerRun(
var buf: [256]u8 = undefined;
const w = std.debug.lockStderrWriter(&buf);
defer std.debug.unlockStderrWriter();
build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ws.ttyconf }, w, false) catch {};
build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ttyconf }, w, false) catch {};
return;
},
else => {
@ -190,20 +217,20 @@ fn fuzzWorkerRun(
}
pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
const gpa = fuzz.ws.gpa;
assert(fuzz.mode == .forever);
var arena_state: std.heap.ArenaAllocator = .init(gpa);
var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
var dedup_table: DedupTable = .empty;
defer dedup_table.deinit(gpa);
defer dedup_table.deinit(fuzz.gpa);
for (fuzz.run_steps) |run_step| {
const compile_inputs = run_step.producer.?.step.inputs.table;
for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len);
try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
for (file_list.items) |sub_path| {
if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
const joined_path = try dir_path.join(arena, sub_path);
@ -224,13 +251,18 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
}
};
std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
return fuzz.ws.serveTarFile(req, deduped_paths);
return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
}
pub const Previous = struct {
unique_runs: usize,
entry_points: usize,
pub const init: Previous = .{ .unique_runs = 0, .entry_points = 0 };
sent_source_index: bool,
pub const init: Previous = .{
.unique_runs = 0,
.entry_points = 0,
.sent_source_index = false,
};
};
pub fn sendUpdate(
fuzz: *Fuzz,
@ -253,7 +285,8 @@ pub fn sendUpdate(
const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
{
if (unique_runs != 0 and prev.unique_runs == 0) {
if (!prev.sent_source_index) {
prev.sent_source_index = true;
// We need to send initial context.
const header: abi.SourceIndexHeader = .{
.directories_len = @intCast(coverage_map.coverage.directories.entries.len),
@ -319,13 +352,13 @@ fn coverageRun(fuzz: *Fuzz) void {
}
}
fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {
const ws = fuzz.ws;
const gpa = ws.gpa;
assert(fuzz.mode == .forever);
const ws = fuzz.mode.forever.ws;
fuzz.coverage_mutex.lock();
defer fuzz.coverage_mutex.unlock();
const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);
const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
if (gop.found_existing) {
// We are fuzzing the same executable with multiple threads.
// Perhaps the same unit test; perhaps a different one. In any
@ -343,16 +376,16 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
.entry_points = .{},
.start_timestamp = ws.now(),
};
errdefer gop.value_ptr.coverage.deinit(gpa);
errdefer gop.value_ptr.coverage.deinit(fuzz.gpa);
const rebuilt_exe_path = run_step.rebuilt_executable.?;
var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
var debug_info = std.debug.Info.load(fuzz.gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
log.err("step '{s}': failed to load debug information for '{f}': {s}", .{
run_step.step.name, rebuilt_exe_path, @errorName(err),
});
return error.AlreadyReported;
};
defer debug_info.deinit(gpa);
defer debug_info.deinit(fuzz.gpa);
const coverage_file_path: Build.Cache.Path = .{
.root_dir = run_step.step.owner.cache_root,
@ -386,14 +419,14 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
const pcs = header.pcAddrs();
const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
errdefer gpa.free(source_locations);
const source_locations = try fuzz.gpa.alloc(Coverage.SourceLocation, pcs.len);
errdefer fuzz.gpa.free(source_locations);
// Unfortunately the PCs array that LLVM gives us from the 8-bit PC
// counters feature is not sorted.
var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};
defer sorted_pcs.deinit(gpa);
try sorted_pcs.resize(gpa, pcs.len);
defer sorted_pcs.deinit(fuzz.gpa);
try sorted_pcs.resize(fuzz.gpa, pcs.len);
@memcpy(sorted_pcs.items(.pc), pcs);
for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
sorted_pcs.sortUnstable(struct {
@ -404,7 +437,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
}
}{ .addrs = sorted_pcs.items(.pc) });
debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
return error.AlreadyReported;
};
@ -414,6 +447,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
ws.notifyUpdate();
}
fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
fuzz.coverage_mutex.lock();
defer fuzz.coverage_mutex.unlock();
@ -445,5 +479,89 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
});
}
try coverage_map.entry_points.append(fuzz.ws.gpa, @intCast(index));
try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
}
pub fn waitAndPrintReport(fuzz: *Fuzz) void {
assert(fuzz.mode == .limit);
fuzz.wait_group.wait();
fuzz.wait_group.reset();
std.debug.print("======= FUZZING REPORT =======\n", .{});
for (fuzz.msg_queue.items) |msg| {
if (msg != .coverage) continue;
const cov = msg.coverage;
const coverage_file_path: std.Build.Cache.Path = .{
.root_dir = cov.run.step.owner.cache_root,
.sub_path = "v/" ++ std.fmt.hex(cov.id),
};
var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
fatal("step '{s}': failed to load coverage file '{f}': {s}", .{
cov.run.step.name, coverage_file_path, @errorName(err),
});
};
defer coverage_file.close();
const fuzz_abi = std.Build.abi.fuzz;
var rbuf: [0x1000]u8 = undefined;
var r = coverage_file.reader(&rbuf);
var header: fuzz_abi.SeenPcsHeader = undefined;
r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
cov.run.step.name, coverage_file_path, @errorName(err),
});
};
if (header.pcs_len == 0) {
fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
cov.run.step.name, coverage_file_path,
});
}
var seen_count: usize = 0;
const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
for (0..chunk_count) |_| {
const seen = r.interface.takeInt(usize, .little) catch |err| {
fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
cov.run.step.name, coverage_file_path, @errorName(err),
});
};
seen_count += @popCount(seen);
}
const seen_f: f64 = @floatFromInt(seen_count);
const total_f: f64 = @floatFromInt(header.pcs_len);
const ratio = seen_f / total_f;
std.debug.print(
\\Step: {s}
\\Fuzz test: "{s}" ({x})
\\Runs: {} -> {}
\\Unique runs: {} -> {}
\\Coverage: {}/{} -> {}/{} ({:.02}%)
\\
, .{
cov.run.step.name,
cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
cov.id,
cov.cumulative.runs,
header.n_runs,
cov.cumulative.unique,
header.unique_runs,
cov.cumulative.coverage,
header.pcs_len,
seen_count,
header.pcs_len,
ratio * 100,
});
std.debug.print("------------------------------\n", .{});
}
std.debug.print(
\\Values are accumulated across multiple runs when preserving the cache.
\\==============================
\\
, .{});
}

View file

@ -1662,12 +1662,24 @@ fn evalZigTest(
// If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has
// somehow already closed; instead, we go straight to capturing stderr in case it has anything
// useful.
const first_write_failed = if (fuzz_context) |fuzz| failed: {
sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index) catch |err| {
try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
break :failed true;
};
break :failed false;
const first_write_failed = if (fuzz_context) |fctx| failed: {
switch (fctx.fuzz.mode) {
.forever => {
const instance_id = 0; // will be used by mutiprocess forever fuzzing
sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .forever, instance_id) catch |err| {
try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
break :failed true;
};
break :failed false;
},
.limit => |limit| {
sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .iterations, limit.amount) catch |err| {
try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
break :failed true;
};
break :failed false;
},
}
} else failed: {
run.fuzz_tests.clearRetainingCapacity();
sendMessage(child.stdin.?, .query_test_metadata) catch |err| {
@ -1778,13 +1790,18 @@ fn evalZigTest(
},
.coverage_id => {
const fuzz = fuzz_context.?.fuzz;
const msg_ptr: *align(1) const u64 = @ptrCast(body);
coverage_id = msg_ptr.*;
const msg_ptr: *align(1) const [4]u64 = @ptrCast(body);
coverage_id = msg_ptr[0];
{
fuzz.queue_mutex.lock();
defer fuzz.queue_mutex.unlock();
try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .coverage = .{
try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
.id = coverage_id.?,
.cumulative = .{
.runs = msg_ptr[1],
.unique = msg_ptr[2],
.coverage = msg_ptr[3],
},
.run = run,
} });
fuzz.queue_cond.signal();
@ -1797,7 +1814,7 @@ fn evalZigTest(
{
fuzz.queue_mutex.lock();
defer fuzz.queue_mutex.unlock();
try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .entry_point = .{
try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
.addr = addr,
.coverage_id = coverage_id.?,
} });
@ -1900,6 +1917,22 @@ fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index:
try file.writeAll(full_msg);
}
fn sendRunFuzzTestMessage(
file: std.fs.File,
index: u32,
kind: std.Build.abi.fuzz.LimitKind,
amount_or_instance: u64,
) !void {
const header: std.zig.Client.Message.Header = .{
.tag = .start_fuzzing,
.bytes_len = 4 + 1 + 8,
};
const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index) ++
std.mem.asBytes(&kind) ++ std.mem.asBytes(&amount_or_instance);
try file.writeAll(full_msg);
}
fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
const b = run.step.owner;
const arena = b.allocator;

View file

@ -219,12 +219,20 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
// Affects or affected by issues #5185, #22523, and #22464.
std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
}
assert(ws.fuzz == null);
ws.build_status.store(.fuzz_init, .monotonic);
ws.notifyUpdate();
ws.fuzz = Fuzz.init(ws) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
ws.fuzz = Fuzz.init(
ws.gpa,
ws.thread_pool,
ws.all_steps,
ws.root_prog_node,
ws.ttyconf,
.{ .forever = .{ .ws = ws } },
) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
ws.fuzz.?.start();
}

View file

@ -140,10 +140,10 @@ pub const Rebuild = extern struct {
pub const fuzz = struct {
pub const TestOne = *const fn (Slice) callconv(.c) void;
pub extern fn fuzzer_init(cache_dir_path: Slice) void;
pub extern fn fuzzer_coverage_id() u64;
pub extern fn fuzzer_coverage() Coverage;
pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void;
pub extern fn fuzzer_new_input(bytes: Slice) void;
pub extern fn fuzzer_main() void;
pub extern fn fuzzer_main(limit_kind: LimitKind, amount: u64) void;
pub const Slice = extern struct {
ptr: [*]const u8,
@ -158,6 +158,8 @@ pub const fuzz = struct {
}
};
pub const LimitKind = enum(u8) { forever, iterations };
/// libfuzzer uses this and its usize is the one that counts. To match the ABI,
/// make the ints be the size of the target used with libfuzzer.
///
@ -251,6 +253,16 @@ pub const fuzz = struct {
return .{ .locs_len_raw = @bitCast(locs_len) };
}
};
/// Sent by lib/fuzzer to test_runner to obtain information about the
/// active memory mapped input file and cumulative stats about previous
/// fuzzing runs.
pub const Coverage = extern struct {
id: u64,
runs: u64,
unique: u64,
seen: u64,
};
};
/// ABI bits specifically relating to the time report interface.

View file

@ -33,10 +33,18 @@ pub const Message = struct {
/// Ask the test runner to run a particular test.
/// The message body is a u32 test index.
run_test,
/// Ask the test runner to start fuzzing a particular test.
/// The message body is a u32 test index.
/// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
/// The message body is:
/// - a u32 test index.
/// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
/// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
start_fuzzing,
_,
};
comptime {
const std = @import("std");
std.debug.assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
}
};

View file

@ -42,9 +42,13 @@ pub const Message = struct {
/// The remaining bytes is the file path relative to that prefix.
/// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
file_system_inputs,
/// Body is a u64le that indicates the file path within the cache used
/// to store coverage information. The integer is a hash of the PCs
/// stored within that file.
/// Body is:
/// - a u64le that indicates the file path within the cache used
/// to store coverage information. The integer is a hash of the PCs
/// stored within that file.
/// - u64le of total runs accumulated
/// - u64le of unique runs accumulated
/// - u64le of coverage accumulated
coverage_id,
/// Body is a u64le that indicates the function pointer virtual memory
/// address of the fuzz unit test. This is used to provide a starting
@ -141,9 +145,15 @@ pub fn receiveMessage(s: *Server) !InMessage.Header {
return s.in.takeStruct(InMessage.Header, .little);
}
pub fn receiveBody_u8(s: *Server) !u8 {
return s.in.takeInt(u8, .little);
}
pub fn receiveBody_u32(s: *Server) !u32 {
return s.in.takeInt(u32, .little);
}
pub fn receiveBody_u64(s: *Server) !u64 {
return s.in.takeInt(u64, .little);
}
pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
try s.serveMessageHeader(.{
@ -160,6 +170,7 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {
}
pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
assert(tag != .coverage_id);
try serveMessageHeader(s, .{
.tag = tag,
.bytes_len = @sizeOf(u64),
@ -168,6 +179,18 @@ pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
try s.out.flush();
}
pub fn serveCoverageIdMessage(s: *const Server, id: u64, runs: u64, unique: u64, cov: u64) !void {
try serveMessageHeader(s, .{
.tag = .coverage_id,
.bytes_len = @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64),
});
try s.out.writeInt(u64, id, .little);
try s.out.writeInt(u64, runs, .little);
try s.out.writeInt(u64, unique, .little);
try s.out.writeInt(u64, cov, .little);
try s.out.flush();
}
pub fn serveEmitDigest(
s: *Server,
digest: *const [Cache.bin_digest_len]u8,

View file

@ -8120,7 +8120,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void {
/// compiler-rt, libcxx, libc, libunwind, etc.
pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
if (comp.debug_compiler_runtime_libs) {
return comp.root_mod.optimize_mode;
return .Debug;
}
const target = &comp.root_mod.resolved_target.result;
switch (comp.root_mod.optimize_mode) {

View file

@ -24,7 +24,7 @@ pub fn main() !void {
abi.fuzzer_new_input(.fromSlice(""));
abi.fuzzer_new_input(.fromSlice("hello"));
const pc_digest = abi.fuzzer_coverage_id();
const pc_digest = abi.fuzzer_coverage().id;
const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest);
const coverage_file = try cache_dir.openFile(coverage_file_path, .{});
defer coverage_file.close();