mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2025-12-06 05:44:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

introduce a web interface for fuzzing

Browse source 
* new .zig-cache subdirectory: 'v'
  - stores coverage information with filename of hash of PCs that want
    coverage. This hash is a hex encoding of the 64-bit coverage ID.
* build runner
  * fixed bug in file system inputs when a compile step has an
    overridden zig_lib_dir field set.
  * set some std lib options optimized for the build runner
    - no side channel mitigations
    - no Transport Layer Security
    - no crypto fork safety
  * add a --port CLI arg for choosing the port the fuzzing web interface
    listens on. it defaults to choosing a random open port.
  * introduce a web server, and serve a basic single page application
    - shares wasm code with autodocs
    - assets are created live on request, for convenient development
      experience. main.wasm is properly cached if nothing changes.
    - sources.tar comes from file system inputs (introduced with the
      `--watch` feature)
  * receives coverage ID from test runner and sends it on a thread-safe
    queue to the WebServer.
* test runner
  - takes a zig cache directory argument now, for where to put coverage
    information.
  - sends coverage ID to parent process
* fuzzer
  - puts its logs (in debug mode) in .zig-cache/tmp/libfuzzer.log
  - computes coverage_id and makes it available with
    `fuzzer_coverage_id` exported function.
  - the memory-mapped coverage file is now namespaced by the coverage id
    in hex encoding, in `.zig-cache/v`
* tokenizer
  - add a fuzz test to check that several properties are upheld
This commit is contained in:
Andrew Kelley 2024-07-31 22:54:05 -07:00
parent ffc050e055
commit e0ffac4e3c
13 changed files with 872 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
lib/compiler/build_runner.zig
View file

@ -17,6 +17,12 @@ const runner = @This();
pub const root = @import("@build");
pub const dependencies = @import("@dependencies");
pub const std_options: std.Options = .{
.side_channels_mitigations = .none,
.http_disable_tls = true,
.crypto_fork_safety = false,
};
pub fn main() !void {
// Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived,
// one shot program. We don't need to waste time freeing memory and finding places to squish
@ -106,6 +112,7 @@ pub fn main() !void {
var watch = false;
var fuzz = false;
var debounce_interval_ms: u16 = 50;
var listen_port: u16 = 0;
while (nextArg(args, &arg_idx)) |arg| {
if (mem.startsWith(u8, arg, "-Z")) {
@ -203,6 +210,14 @@ pub fn main() !void {
next_arg, @errorName(err),
});
};
} else if (mem.eql(u8, arg, "--port")) {
const next_arg = nextArg(args, &arg_idx) orelse
fatalWithHint("expected u16 after '{s}'", .{arg});
listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| {
fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{
next_arg, @errorName(err),
});
};
} else if (mem.eql(u8, arg, "--debug-log")) {
const next_arg = nextArgOrFatal(args, &arg_idx);
try debug_log_scopes.append(next_arg);
@ -403,7 +418,19 @@ pub fn main() !void {
else => return err,
};
if (fuzz) {
Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node);
const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable;
try Fuzz.start(
gpa,
arena,
global_cache_directory,
zig_lib_directory,
zig_exe,
&run.thread_pool,
run.step_stack.keys(),
run.ttyconf,
listen_address,
main_progress_node,
);
}
if (!watch) return cleanExit();

22
lib/compiler/test_runner.zig
View file

@ -28,6 +28,7 @@ pub fn main() void {
@panic("unable to parse command line args");
var listen = false;
var opt_cache_dir: ?[]const u8 = null;
for (args[1..]) |arg| {
if (std.mem.eql(u8, arg, "--listen=-")) {
@ -35,13 +36,18 @@ pub fn main() void {
} else if (std.mem.startsWith(u8, arg, "--seed=")) {
testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
@panic("unable to parse --seed command line argument");
} else if (std.mem.startsWith(u8, arg, "--cache-dir")) {
opt_cache_dir = arg["--cache-dir=".len..];
} else {
@panic("unrecognized command line argument");
}
}
fba.reset();
if (builtin.fuzz) fuzzer_init();
if (builtin.fuzz) {
const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
fuzzer_init(FuzzerSlice.fromSlice(cache_dir));
}
if (listen) {
return mainServer() catch @panic("internal test runner failure");
@ -60,6 +66,11 @@ fn mainServer() !void {
});
defer server.deinit();
if (builtin.fuzz) {
const coverage_id = fuzzer_coverage_id();
try server.serveU64Message(.coverage_id, coverage_id);
}
while (true) {
const hdr = try server.receiveMessage();
switch (hdr.tag) {
@ -316,15 +327,22 @@ const FuzzerSlice = extern struct {
ptr: [*]const u8,
len: usize,
/// Inline to avoid fuzzer instrumentation.
inline fn toSlice(s: FuzzerSlice) []const u8 {
return s.ptr[0..s.len];
}
/// Inline to avoid fuzzer instrumentation.
inline fn fromSlice(s: []const u8) FuzzerSlice {
return .{ .ptr = s.ptr, .len = s.len };
}
};
var is_fuzz_test: bool = undefined;
extern fn fuzzer_next() FuzzerSlice;
extern fn fuzzer_init() void;
extern fn fuzzer_init(cache_dir: FuzzerSlice) void;
extern fn fuzzer_coverage_id() u64;
pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
@disableInstrumentation();

4
lib/docs/wasm/main.zig
View file

@ -53,7 +53,7 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
const tar_bytes = tar_ptr[0..tar_len];
//log.debug("received {d} bytes of tar file", .{tar_bytes.len});
unpack_inner(tar_bytes) catch |err| {
unpackInner(tar_bytes) catch |err| {
fatal("unable to unpack tar: {s}", .{@errorName(err)});
};
}
@ -750,7 +750,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {
const Oom = error{OutOfMemory};
fn unpack_inner(tar_bytes: []u8) !void {
fn unpackInner(tar_bytes: []u8) !void {
var fbs = std.io.fixedBufferStream(tar_bytes);
var file_name_buffer: [1024]u8 = undefined;
var link_name_buffer: [1024]u8 = undefined;

73
lib/fuzzer.zig
View file

@ -17,7 +17,8 @@ fn logOverride(
args: anytype,
) void {
const f = if (log_file) |f| f else f: {
const f = fuzzer.dir.createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file");
const f = fuzzer.cache_dir.createFile("tmp/libfuzzer.log", .{}) catch
@panic("failed to open fuzzer log file");
log_file = f;
break :f f;
};
@ -114,7 +115,10 @@ const Fuzzer = struct {
/// Stored in a memory-mapped file so that it can be shared with other
/// processes and viewed while the fuzzer is running.
seen_pcs: MemoryMappedList,
dir: std.fs.Dir,
cache_dir: std.fs.Dir,
/// Identifies the file name that will be used to store coverage
/// information, available to other processes.
coverage_id: u64,
const SeenPcsHeader = extern struct {
n_runs: usize,
@ -189,18 +193,31 @@ const Fuzzer = struct {
id: Run.Id,
};
fn init(f: *Fuzzer, dir: std.fs.Dir) !void {
f.dir = dir;
fn init(f: *Fuzzer, cache_dir: std.fs.Dir) !void {
const flagged_pcs = f.flagged_pcs;
f.cache_dir = cache_dir;
// Choose a file name for the coverage based on a hash of the PCs that will be stored within.
const pc_digest = d: {
var hasher = std.hash.Wyhash.init(0);
for (flagged_pcs) |flagged_pc| {
hasher.update(std.mem.asBytes(&flagged_pc.addr));
}
break :d f.coverage.run_id_hasher.final();
};
f.coverage_id = pc_digest;
const hex_digest = std.fmt.hex(pc_digest);
const coverage_file_path = "v/" ++ hex_digest;
// Layout of this file:
// - Header
// - list of PC addresses (usize elements)
// - list of hit flag, 1 bit per address (stored in u8 elements)
const coverage_file = dir.createFile("coverage", .{
const coverage_file = createFileBail(cache_dir, coverage_file_path, .{
.read = true,
.truncate = false,
}) catch |err| fatal("unable to create coverage file: {s}", .{@errorName(err)});
const flagged_pcs = f.flagged_pcs;
});
const n_bitset_elems = (flagged_pcs.len + 7) / 8;
const bytes_len = @sizeOf(SeenPcsHeader) + flagged_pcs.len * @sizeOf(usize) + n_bitset_elems;
const existing_len = coverage_file.getEndPos() catch |err| {
@ -217,7 +234,8 @@ const Fuzzer = struct {
fatal("unable to init coverage memory map: {s}", .{@errorName(err)});
};
if (existing_len != 0) {
const existing_pcs = std.mem.bytesAsSlice(usize, f.seen_pcs.items[@sizeOf(SeenPcsHeader)..][0 .. flagged_pcs.len * @sizeOf(usize)]);
const existing_pcs_bytes = f.seen_pcs.items[@sizeOf(SeenPcsHeader)..][0 .. flagged_pcs.len * @sizeOf(usize)];
const existing_pcs = std.mem.bytesAsSlice(usize, existing_pcs_bytes);
for (existing_pcs, flagged_pcs, 0..) |old, new, i| {
if (old != new.addr) {
fatal("incompatible existing coverage file (differing PC at index {d}: {x} != {x})", .{
@ -380,6 +398,21 @@ const Fuzzer = struct {
}
};
fn createFileBail(dir: std.fs.Dir, sub_path: []const u8, flags: std.fs.File.CreateFlags) std.fs.File {
return dir.createFile(sub_path, flags) catch |err| switch (err) {
error.FileNotFound => {
const dir_name = std.fs.path.dirname(sub_path).?;
dir.makePath(dir_name) catch |e| {
fatal("unable to make path '{s}': {s}", .{ dir_name, @errorName(e) });
};
return dir.createFile(sub_path, flags) catch |e| {
fatal("unable to create file '{s}': {s}", .{ sub_path, @errorName(e) });
};
},
else => fatal("unable to create file '{s}': {s}", .{ sub_path, @errorName(err) }),
};
}
fn oom(err: anytype) noreturn {
switch (err) {
error.OutOfMemory => @panic("out of memory"),
@ -397,25 +430,35 @@ var fuzzer: Fuzzer = .{
.n_runs = 0,
.recent_cases = .{},
.coverage = undefined,
.dir = undefined,
.cache_dir = undefined,
.seen_pcs = undefined,
.coverage_id = undefined,
};
/// Invalid until `fuzzer_init` is called.
export fn fuzzer_coverage_id() u64 {
return fuzzer.coverage_id;
}
export fn fuzzer_next() Fuzzer.Slice {
return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) {
error.OutOfMemory => @panic("out of memory"),
});
}
export fn fuzzer_init() void {
export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void {
if (module_count_8bc == 0) fatal("__sanitizer_cov_8bit_counters_init was never called", .{});
if (module_count_pcs == 0) fatal("__sanitizer_cov_pcs_init was never called", .{});
// TODO: move this to .zig-cache/f
const fuzz_dir = std.fs.cwd().makeOpenPath("f", .{ .iterate = true }) catch |err| {
fatal("unable to open fuzz directory 'f': {s}", .{@errorName(err)});
};
fuzzer.init(fuzz_dir) catch |err| fatal("unable to init fuzzer: {s}", .{@errorName(err)});
const cache_dir_path = cache_dir_struct.toZig();
const cache_dir = if (cache_dir_path.len == 0)
std.fs.cwd()
else
std.fs.cwd().makeOpenPath(cache_dir_path, .{ .iterate = true }) catch |err| {
fatal("unable to open fuzz directory '{s}': {s}", .{ cache_dir_path, @errorName(err) });
};
fuzzer.init(cache_dir) catch |err| fatal("unable to init fuzzer: {s}", .{@errorName(err)});
}
/// Like `std.ArrayListUnmanaged(u8)` but backed by memory mapping.

76
lib/fuzzer/index.html Normal file
View file

@ -0,0 +1,76 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<title>Zig Documentation</title>
<style type="text/css">
body {
font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;
color: #000000;
}
.tok-kw {
color: #333;
font-weight: bold;
}
.tok-str {
color: #d14;
}
.tok-builtin {
color: #0086b3;
}
.tok-comment {
color: #777;
font-style: italic;
}
.tok-fn {
color: #900;
font-weight: bold;
}
.tok-null {
color: #008080;
}
.tok-number {
color: #008080;
}
.tok-type {
color: #458;
font-weight: bold;
}
@media (prefers-color-scheme: dark) {
body {
background-color: #111;
color: #bbb;
}
.tok-kw {
color: #eee;
}
.tok-str {
color: #2e5;
}
.tok-builtin {
color: #ff894c;
}
.tok-comment {
color: #aa7;
}
.tok-fn {
color: #B1A0F8;
}
.tok-null {
color: #ff8080;
}
.tok-number {
color: #ff8080;
}
.tok-type {
color: #68f;
}
}
</style>
</head>
<body>
<script src="main.js"></script>
</body>
</html>

40
lib/fuzzer/main.js Normal file
View file

@ -0,0 +1,40 @@
(function() {
let wasm_promise = fetch("main.wasm");
let sources_promise = fetch("sources.tar").then(function(response) {
if (!response.ok) throw new Error("unable to download sources");
return response.arrayBuffer();
});
var wasm_exports = null;
const text_decoder = new TextDecoder();
const text_encoder = new TextEncoder();
WebAssembly.instantiateStreaming(wasm_promise, {
js: {
log: function(ptr, len) {
const msg = decodeString(ptr, len);
console.log(msg);
},
panic: function (ptr, len) {
const msg = decodeString(ptr, len);
throw new Error("panic: " + msg);
},
},
}).then(function(obj) {
wasm_exports = obj.instance.exports;
window.wasm = obj; // for debugging
sources_promise.then(function(buffer) {
const js_array = new Uint8Array(buffer);
const ptr = wasm_exports.alloc(js_array.length);
const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
wasm_array.set(js_array);
wasm_exports.unpack(ptr, js_array.length);
});
});
function decodeString(ptr, len) {
if (len === 0) return "";
return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
}
})();

99
lib/fuzzer/wasm/main.zig Normal file
View file

@ -0,0 +1,99 @@
const std = @import("std");
const assert = std.debug.assert;
const Walk = @import("Walk");
const gpa = std.heap.wasm_allocator;
const log = std.log;
const js = struct {
extern "js" fn log(ptr: [*]const u8, len: usize) void;
extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;
};
pub const std_options: std.Options = .{
.logFn = logFn,
};
pub fn panic(msg: []const u8, st: ?*std.builtin.StackTrace, addr: ?usize) noreturn {
_ = st;
_ = addr;
log.err("panic: {s}", .{msg});
@trap();
}
fn logFn(
comptime message_level: log.Level,
comptime scope: @TypeOf(.enum_literal),
comptime format: []const u8,
args: anytype,
) void {
const level_txt = comptime message_level.asText();
const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
var buf: [500]u8 = undefined;
const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: {
buf[buf.len - 3 ..][0..3].* = "...".*;
break :l &buf;
};
js.log(line.ptr, line.len);
}
export fn alloc(n: usize) [*]u8 {
const slice = gpa.alloc(u8, n) catch @panic("OOM");
return slice.ptr;
}
export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
const tar_bytes = tar_ptr[0..tar_len];
log.debug("received {d} bytes of tar file", .{tar_bytes.len});
unpackInner(tar_bytes) catch |err| {
fatal("unable to unpack tar: {s}", .{@errorName(err)});
};
}
fn unpackInner(tar_bytes: []u8) !void {
var fbs = std.io.fixedBufferStream(tar_bytes);
var file_name_buffer: [1024]u8 = undefined;
var link_name_buffer: [1024]u8 = undefined;
var it = std.tar.iterator(fbs.reader(), .{
.file_name_buffer = &file_name_buffer,
.link_name_buffer = &link_name_buffer,
});
while (try it.next()) |tar_file| {
switch (tar_file.kind) {
.file => {
if (tar_file.size == 0 and tar_file.name.len == 0) break;
if (std.mem.endsWith(u8, tar_file.name, ".zig")) {
log.debug("found file: '{s}'", .{tar_file.name});
const file_name = try gpa.dupe(u8, tar_file.name);
if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| {
const pkg_name = file_name[0..pkg_name_end];
const gop = try Walk.modules.getOrPut(gpa, pkg_name);
const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len);
if (!gop.found_existing or
std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or
std.mem.eql(u8, file_name[pkg_name_end + 1 .. file_name.len - ".zig".len], pkg_name))
{
gop.value_ptr.* = file;
}
const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)];
assert(file == try Walk.add_file(file_name, file_bytes));
}
} else {
log.warn("skipping: '{s}' - the tar creation should have done that", .{tar_file.name});
}
},
else => continue,
}
}
}
fn fatal(comptime format: []const u8, args: anytype) noreturn {
var buf: [500]u8 = undefined;
const line = std.fmt.bufPrint(&buf, format, args) catch l: {
buf[buf.len - 3 ..][0..3].* = "...".*;
break :l &buf;
};
js.panic(line.ptr, line.len);
}

12
lib/std/Build.zig
View file

@ -2300,22 +2300,26 @@ pub const LazyPath = union(enum) {
}
pub fn path(lazy_path: LazyPath, b: *Build, sub_path: []const u8) LazyPath {
return lazy_path.join(b.allocator, sub_path) catch @panic("OOM");
}
pub fn join(lazy_path: LazyPath, arena: Allocator, sub_path: []const u8) Allocator.Error!LazyPath {
return switch (lazy_path) {
.src_path => |src| .{ .src_path = .{
.owner = src.owner,
.sub_path = b.pathResolve(&.{ src.sub_path, sub_path }),
.sub_path = try fs.path.resolve(arena, &.{ src.sub_path, sub_path }),
} },
.generated => |gen| .{ .generated = .{
.file = gen.file,
.up = gen.up,
.sub_path = b.pathResolve(&.{ gen.sub_path, sub_path }),
.sub_path = try fs.path.resolve(arena, &.{ gen.sub_path, sub_path }),
} },
.cwd_relative => |cwd_relative| .{
.cwd_relative = b.pathResolve(&.{ cwd_relative, sub_path }),
.cwd_relative = try fs.path.resolve(arena, &.{ cwd_relative, sub_path }),
},
.dependency => |dep| .{ .dependency = .{
.dependency = dep.dependency,
.sub_path = b.pathResolve(&.{ dep.sub_path, sub_path }),
.sub_path = try fs.path.resolve(arena, &.{ dep.sub_path, sub_path }),
} },
};
}

455
lib/std/Build/Fuzz.zig
View file

@ -1,59 +1,479 @@
const builtin = @import("builtin");
const std = @import("../std.zig");
const Fuzz = @This();
const Build = std.Build;
const Step = std.Build.Step;
const assert = std.debug.assert;
const fatal = std.process.fatal;
const Allocator = std.mem.Allocator;
const log = std.log;
const Fuzz = @This();
const build_runner = @import("root");
pub fn start(
gpa: Allocator,
arena: Allocator,
global_cache_directory: Build.Cache.Directory,
zig_lib_directory: Build.Cache.Directory,
zig_exe_path: []const u8,
thread_pool: *std.Thread.Pool,
all_steps: []const *Step,
ttyconf: std.io.tty.Config,
listen_address: std.net.Address,
prog_node: std.Progress.Node,
) void {
const count = block: {
) Allocator.Error!void {
const fuzz_run_steps = block: {
const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
defer rebuild_node.end();
var count: usize = 0;
var wait_group: std.Thread.WaitGroup = .{};
defer wait_group.wait();
var fuzz_run_steps: std.ArrayListUnmanaged(*Step.Run) = .{};
defer fuzz_run_steps.deinit(gpa);
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
if (run.fuzz_tests.items.len > 0 and run.producer != null) {
thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node });
count += 1;
try fuzz_run_steps.append(gpa, run);
}
}
if (count == 0) fatal("no fuzz tests found", .{});
rebuild_node.setEstimatedTotalItems(count);
break :block count;
if (fuzz_run_steps.items.len == 0) fatal("no fuzz tests found", .{});
rebuild_node.setEstimatedTotalItems(fuzz_run_steps.items.len);
break :block try arena.dupe(*Step.Run, fuzz_run_steps.items);
};
// Detect failure.
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
if (run.fuzz_tests.items.len > 0 and run.rebuilt_executable == null)
for (fuzz_run_steps) |run| {
assert(run.fuzz_tests.items.len > 0);
if (run.rebuilt_executable == null)
fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
}
var web_server: WebServer = .{
.gpa = gpa,
.global_cache_directory = global_cache_directory,
.zig_lib_directory = zig_lib_directory,
.zig_exe_path = zig_exe_path,
.msg_queue = .{},
.mutex = .{},
.listen_address = listen_address,
.fuzz_run_steps = fuzz_run_steps,
};
const web_server_thread = std.Thread.spawn(.{}, WebServer.run, .{&web_server}) catch |err| {
fatal("unable to spawn web server thread: {s}", .{@errorName(err)});
};
defer web_server_thread.join();
{
const fuzz_node = prog_node.start("Fuzzing", count);
const fuzz_node = prog_node.start("Fuzzing", fuzz_run_steps.len);
defer fuzz_node.end();
var wait_group: std.Thread.WaitGroup = .{};
defer wait_group.wait();
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
for (fuzz_run_steps) |run| {
for (run.fuzz_tests.items) |unit_test_index| {
assert(run.rebuilt_executable != null);
thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, fuzz_node });
thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{
run, &web_server, unit_test_index, ttyconf, fuzz_node,
});
}
}
}
fatal("all fuzz workers crashed", .{});
log.err("all fuzz workers crashed", .{});
}
pub const WebServer = struct {
gpa: Allocator,
global_cache_directory: Build.Cache.Directory,
zig_lib_directory: Build.Cache.Directory,
zig_exe_path: []const u8,
/// Messages from fuzz workers. Protected by mutex.
msg_queue: std.ArrayListUnmanaged(Msg),
mutex: std.Thread.Mutex,
listen_address: std.net.Address,
fuzz_run_steps: []const *Step.Run,
const Msg = union(enum) {
coverage_id: u64,
};
fn run(ws: *WebServer) void {
var http_server = ws.listen_address.listen(.{
.reuse_address = true,
}) catch |err| {
log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) });
return;
};
const port = http_server.listen_address.in.getPort();
log.info("web interface listening at http://127.0.0.1:{d}/", .{port});
while (true) {
const connection = http_server.accept() catch |err| {
log.err("failed to accept connection: {s}", .{@errorName(err)});
return;
};
_ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch |err| {
log.err("unable to spawn connection thread: {s}", .{@errorName(err)});
connection.stream.close();
continue;
};
}
}
fn accept(ws: *WebServer, connection: std.net.Server.Connection) void {
defer connection.stream.close();
var read_buffer: [8000]u8 = undefined;
var server = std.http.Server.init(connection, &read_buffer);
while (server.state == .ready) {
var request = server.receiveHead() catch |err| switch (err) {
error.HttpConnectionClosing => return,
else => {
log.err("closing http connection: {s}", .{@errorName(err)});
return;
},
};
serveRequest(ws, &request) catch |err| switch (err) {
error.AlreadyReported => return,
else => |e| {
log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) });
return;
},
};
}
}
fn serveRequest(ws: *WebServer, request: *std.http.Server.Request) !void {
if (std.mem.eql(u8, request.head.target, "/") or
std.mem.eql(u8, request.head.target, "/debug") or
std.mem.eql(u8, request.head.target, "/debug/"))
{
try serveFile(ws, request, "fuzzer/index.html", "text/html");
} else if (std.mem.eql(u8, request.head.target, "/main.js") or
std.mem.eql(u8, request.head.target, "/debug/main.js"))
{
try serveFile(ws, request, "fuzzer/main.js", "application/javascript");
} else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
try serveWasm(ws, request, .ReleaseFast);
} else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
try serveWasm(ws, request, .Debug);
} else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
{
try serveSourcesTar(ws, request);
} else {
try request.respond("not found", .{
.status = .not_found,
.extra_headers = &.{
.{ .name = "content-type", .value = "text/plain" },
},
});
}
}
fn serveFile(
ws: *WebServer,
request: *std.http.Server.Request,
name: []const u8,
content_type: []const u8,
) !void {
const gpa = ws.gpa;
// The desired API is actually sendfile, which will require enhancing std.http.Server.
// We load the file with every request so that the user can make changes to the file
// and refresh the HTML page without restarting this server.
const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch |err| {
log.err("failed to read '{}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) });
return error.AlreadyReported;
};
defer gpa.free(file_contents);
try request.respond(file_contents, .{
.extra_headers = &.{
.{ .name = "content-type", .value = content_type },
cache_control_header,
},
});
}
fn serveWasm(
ws: *WebServer,
request: *std.http.Server.Request,
optimize_mode: std.builtin.OptimizeMode,
) !void {
const gpa = ws.gpa;
var arena_instance = std.heap.ArenaAllocator.init(gpa);
defer arena_instance.deinit();
const arena = arena_instance.allocator();
// Do the compilation every request, so that the user can edit the files
// and see the changes without restarting the server.
const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
// std.http.Server does not have a sendfile API yet.
const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
defer gpa.free(file_contents);
try request.respond(file_contents, .{
.extra_headers = &.{
.{ .name = "content-type", .value = "application/wasm" },
cache_control_header,
},
});
}
fn buildWasmBinary(
ws: *WebServer,
arena: Allocator,
optimize_mode: std.builtin.OptimizeMode,
) ![]const u8 {
const gpa = ws.gpa;
const main_src_path: Build.Cache.Path = .{
.root_dir = ws.zig_lib_directory,
.sub_path = "fuzzer/wasm/main.zig",
};
const walk_src_path: Build.Cache.Path = .{
.root_dir = ws.zig_lib_directory,
.sub_path = "docs/wasm/Walk.zig",
};
var argv: std.ArrayListUnmanaged([]const u8) = .{};
try argv.appendSlice(arena, &.{
ws.zig_exe_path,
"build-exe",
"-fno-entry",
"-O",
@tagName(optimize_mode),
"-target",
"wasm32-freestanding",
"-mcpu",
"baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",
"--cache-dir",
ws.global_cache_directory.path orelse ".",
"--global-cache-dir",
ws.global_cache_directory.path orelse ".",
"--name",
"fuzzer",
"-rdynamic",
"--dep",
"Walk",
try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}),
try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}),
"--listen=-",
});
var child = std.process.Child.init(argv.items, gpa);
child.stdin_behavior = .Pipe;
child.stdout_behavior = .Pipe;
child.stderr_behavior = .Pipe;
try child.spawn();
var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
.stdout = child.stdout.?,
.stderr = child.stderr.?,
});
defer poller.deinit();
try sendMessage(child.stdin.?, .update);
try sendMessage(child.stdin.?, .exit);
const Header = std.zig.Server.Message.Header;
var result: ?[]const u8 = null;
var result_error_bundle = std.zig.ErrorBundle.empty;
const stdout = poller.fifo(.stdout);
poll: while (true) {
while (stdout.readableLength() < @sizeOf(Header)) {
if (!(try poller.poll())) break :poll;
}
const header = stdout.reader().readStruct(Header) catch unreachable;
while (stdout.readableLength() < header.bytes_len) {
if (!(try poller.poll())) break :poll;
}
const body = stdout.readableSliceOfLen(header.bytes_len);
switch (header.tag) {
.zig_version => {
if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
return error.ZigProtocolVersionMismatch;
}
},
.error_bundle => {
const EbHdr = std.zig.Server.Message.ErrorBundle;
const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
const extra_bytes =
body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
const string_bytes =
body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
// TODO: use @ptrCast when the compiler supports it
const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
const extra_array = try arena.alloc(u32, unaligned_extra.len);
@memcpy(extra_array, unaligned_extra);
result_error_bundle = .{
.string_bytes = try arena.dupe(u8, string_bytes),
.extra = extra_array,
};
},
.emit_bin_path => {
const EbpHdr = std.zig.Server.Message.EmitBinPath;
const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
if (!ebp_hdr.flags.cache_hit) {
log.info("source changes detected; rebuilt wasm component", .{});
}
result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
},
else => {}, // ignore other messages
}
stdout.discard(body.len);
}
const stderr = poller.fifo(.stderr);
if (stderr.readableLength() > 0) {
const owned_stderr = try stderr.toOwnedSlice();
defer gpa.free(owned_stderr);
std.debug.print("{s}", .{owned_stderr});
}
// Send EOF to stdin.
child.stdin.?.close();
child.stdin = null;
switch (try child.wait()) {
.Exited => |code| {
if (code != 0) {
log.err(
"the following command exited with error code {d}:\n{s}",
.{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) },
);
return error.WasmCompilationFailed;
}
},
.Signal, .Stopped, .Unknown => {
log.err(
"the following command terminated unexpectedly:\n{s}",
.{try Build.Step.allocPrintCmd(arena, null, argv.items)},
);
return error.WasmCompilationFailed;
},
}
if (result_error_bundle.errorMessageCount() > 0) {
const color = std.zig.Color.auto;
result_error_bundle.renderToStdErr(color.renderOptions());
log.err("the following command failed with {d} compilation errors:\n{s}", .{
result_error_bundle.errorMessageCount(),
try Build.Step.allocPrintCmd(arena, null, argv.items),
});
return error.WasmCompilationFailed;
}
return result orelse {
log.err("child process failed to report result\n{s}", .{
try Build.Step.allocPrintCmd(arena, null, argv.items),
});
return error.WasmCompilationFailed;
};
}
fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
const header: std.zig.Client.Message.Header = .{
.tag = tag,
.bytes_len = 0,
};
try file.writeAll(std.mem.asBytes(&header));
}
fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {
const gpa = ws.gpa;
var arena_instance = std.heap.ArenaAllocator.init(gpa);
defer arena_instance.deinit();
const arena = arena_instance.allocator();
var send_buffer: [0x4000]u8 = undefined;
var response = request.respondStreaming(.{
.send_buffer = &send_buffer,
.respond_options = .{
.extra_headers = &.{
.{ .name = "content-type", .value = "application/x-tar" },
cache_control_header,
},
},
});
const w = response.writer();
const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
var dedupe_table: DedupeTable = .{};
defer dedupe_table.deinit(gpa);
for (ws.fuzz_run_steps) |run_step| {
const compile_step_inputs = run_step.producer.?.step.inputs.table;
for (compile_step_inputs.keys(), compile_step_inputs.values()) |dir_path, *file_list| {
try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len);
for (file_list.items) |sub_path| {
// Special file "." means the entire directory.
if (std.mem.eql(u8, sub_path, ".")) continue;
const joined_path = try dir_path.join(arena, sub_path);
_ = dedupe_table.getOrPutAssumeCapacity(joined_path);
}
}
}
const deduped_paths = dedupe_table.keys();
const SortContext = struct {
pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
_ = this;
return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
.lt => true,
.gt => false,
.eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
};
}
};
std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
for (deduped_paths) |joined_path| {
var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch |err| {
log.err("failed to open {}: {s}", .{ joined_path, @errorName(err) });
continue;
};
defer file.close();
const stat = file.stat() catch |err| {
log.err("failed to stat {}: {s}", .{ joined_path, @errorName(err) });
continue;
};
if (stat.kind != .file)
continue;
const padding = p: {
const remainder = stat.size % 512;
break :p if (remainder > 0) 512 - remainder else 0;
};
var file_header = std.tar.output.Header.init();
file_header.typeflag = .regular;
try file_header.setPath(joined_path.root_dir.path orelse ".", joined_path.sub_path);
try file_header.setSize(stat.size);
try file_header.updateChecksum();
try w.writeAll(std.mem.asBytes(&file_header));
try w.writeFile(file);
try w.writeByteNTimes(0, padding);
}
// intentionally omitting the pointless trailer
//try w.writeByteNTimes(0, 512 * 2);
try response.end();
}
const cache_control_header: std.http.Header = .{
.name = "cache-control",
.value = "max-age=0, must-revalidate",
};
};
fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
const gpa = run.step.owner.allocator;
const stderr = std.io.getStdErr();
@ -88,6 +508,7 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog
fn fuzzWorkerRun(
run: *Step.Run,
web_server: *WebServer,
unit_test_index: u32,
ttyconf: std.io.tty.Config,
parent_prog_node: std.Progress.Node,
@ -98,7 +519,7 @@ fn fuzzWorkerRun(
const prog_node = parent_prog_node.start(test_name, 0);
defer prog_node.end();
run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) {
run.rerunInFuzzMode(web_server, unit_test_index, prog_node) catch |err| switch (err) {
error.MakeFailed => {
const stderr = std.io.getStdErr();
std.debug.lockStdErr();

3
lib/std/Build/Step.zig
View file

@ -559,7 +559,8 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
},
.zig_lib => zl: {
if (s.cast(Step.Compile)) |compile| {
if (compile.zig_lib_dir) |lp| {
if (compile.zig_lib_dir) |zig_lib_dir| {
const lp = try zig_lib_dir.join(arena, sub_path);
try addWatchInput(s, lp);
break :zl;
}

52
lib/std/Build/Step/Run.zig
View file

@ -205,6 +205,7 @@ pub fn enableTestRunnerMode(run: *Run) void {
run.stdio = .zig_test;
run.addArgs(&.{
std.fmt.allocPrint(arena, "--seed=0x{x}", .{b.graph.random_seed}) catch @panic("OOM"),
std.fmt.allocPrint(arena, "--cache-dir={s}", .{b.cache_root.path orelse ""}) catch @panic("OOM"),
"--listen=-",
});
}
@ -845,7 +846,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
);
}
pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void {
pub fn rerunInFuzzMode(
run: *Run,
web_server: *std.Build.Fuzz.WebServer,
unit_test_index: u32,
prog_node: std.Progress.Node,
) !void {
const step = &run.step;
const b = step.owner;
const arena = b.allocator;
@ -877,7 +883,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.
const has_side_effects = false;
const rand_int = std.crypto.random.int(u64);
const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index);
try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, .{
.unit_test_index = unit_test_index,
.web_server = web_server,
});
}
fn populateGeneratedPaths(
@ -952,13 +961,18 @@ fn termMatches(expected: ?std.process.Child.Term, actual: std.process.Child.Term
};
}
const FuzzContext = struct {
web_server: *std.Build.Fuzz.WebServer,
unit_test_index: u32,
};
fn runCommand(
run: *Run,
argv: []const []const u8,
has_side_effects: bool,
output_dir_path: []const u8,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
fuzz_context: ?FuzzContext,
) !void {
const step = &run.step;
const b = step.owner;
@ -977,7 +991,7 @@ fn runCommand(
var interp_argv = std.ArrayList([]const u8).init(b.allocator);
defer interp_argv.deinit();
const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: {
const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_context) catch |err| term: {
// InvalidExe: cpu arch mismatch
// FileNotFound: can happen with a wrong dynamic linker path
if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
@ -1113,7 +1127,7 @@ fn runCommand(
try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items);
break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| {
break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_context) catch |e| {
if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
return step.fail("unable to spawn interpreter {s}: {s}", .{
@ -1133,7 +1147,7 @@ fn runCommand(
const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
if (fuzz_unit_test_index != null) {
if (fuzz_context != null) {
try step.handleChildProcessTerm(result.term, cwd, final_argv);
return;
}
@ -1298,12 +1312,12 @@ fn spawnChildAndCollect(
argv: []const []const u8,
has_side_effects: bool,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
fuzz_context: ?FuzzContext,
) !ChildProcResult {
const b = run.step.owner;
const arena = b.allocator;
if (fuzz_unit_test_index != null) {
if (fuzz_context != null) {
assert(!has_side_effects);
assert(run.stdio == .zig_test);
}
@ -1357,7 +1371,7 @@ fn spawnChildAndCollect(
var timer = try std.time.Timer.start();
const result = if (run.stdio == .zig_test)
evalZigTest(run, &child, prog_node, fuzz_unit_test_index)
evalZigTest(run, &child, prog_node, fuzz_context)
else
evalGeneric(run, &child);
@ -1383,7 +1397,7 @@ fn evalZigTest(
run: *Run,
child: *std.process.Child,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
fuzz_context: ?FuzzContext,
) !StdIoResult {
const gpa = run.step.owner.allocator;
const arena = run.step.owner.allocator;
@ -1394,8 +1408,8 @@ fn evalZigTest(
});
defer poller.deinit();
if (fuzz_unit_test_index) |index| {
try sendRunTestMessage(child.stdin.?, .start_fuzzing, index);
if (fuzz_context) |fuzz| {
try sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index);
} else {
run.fuzz_tests.clearRetainingCapacity();
try sendMessage(child.stdin.?, .query_test_metadata);
@ -1437,7 +1451,7 @@ fn evalZigTest(
}
},
.test_metadata => {
assert(fuzz_unit_test_index == null);
assert(fuzz_context == null);
const TmHdr = std.zig.Server.Message.TestMetadata;
const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));
test_count = tm_hdr.tests_len;
@ -1466,7 +1480,7 @@ fn evalZigTest(
try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node);
},
.test_results => {
assert(fuzz_unit_test_index == null);
assert(fuzz_context == null);
const md = metadata.?;
const TrHdr = std.zig.Server.Message.TestResults;
@ -1500,6 +1514,16 @@ fn evalZigTest(
try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node);
},
.coverage_id => {
const web_server = fuzz_context.?.web_server;
const msg_ptr: *align(1) const u64 = @ptrCast(body);
const coverage_id = msg_ptr.*;
{
web_server.mutex.lock();
defer web_server.mutex.unlock();
try web_server.msg_queue.append(web_server.gpa, .{ .coverage_id = coverage_id });
}
},
else => {}, // ignore other messages
}

24
lib/std/zig/Server.zig
View file

@ -28,6 +28,10 @@ pub const Message = struct {
/// The remaining bytes is the file path relative to that prefix.
/// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
file_system_inputs,
/// Body is a u64le that indicates the file path within the cache used
/// to store coverage information. The integer is a hash of the PCs
/// stored within that file.
coverage_id,
_,
};
@ -180,6 +184,14 @@ pub fn serveMessage(
try s.out.writevAll(iovecs[0 .. bufs.len + 1]);
}
pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void {
const msg_le = bswap(int);
return s.serveMessage(.{
.tag = tag,
.bytes_len = @sizeOf(u64),
}, &.{std.mem.asBytes(&msg_le)});
}
pub fn serveEmitBinPath(
s: *Server,
fs_path: []const u8,
@ -187,7 +199,7 @@ pub fn serveEmitBinPath(
) !void {
try s.serveMessage(.{
.tag = .emit_bin_path,
.bytes_len = @as(u32, @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath))),
.bytes_len = @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath)),
}, &.{
std.mem.asBytes(&header),
fs_path,
@ -201,7 +213,7 @@ pub fn serveTestResults(
const msg_le = bswap(msg);
try s.serveMessage(.{
.tag = .test_results,
.bytes_len = @as(u32, @intCast(@sizeOf(OutMessage.TestResults))),
.bytes_len = @intCast(@sizeOf(OutMessage.TestResults)),
}, &.{
std.mem.asBytes(&msg_le),
});
@ -209,14 +221,14 @@ pub fn serveTestResults(
pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void {
const eb_hdr: OutMessage.ErrorBundle = .{
.extra_len = @as(u32, @intCast(error_bundle.extra.len)),
.string_bytes_len = @as(u32, @intCast(error_bundle.string_bytes.len)),
.extra_len = @intCast(error_bundle.extra.len),
.string_bytes_len = @intCast(error_bundle.string_bytes.len),
};
const bytes_len = @sizeOf(OutMessage.ErrorBundle) +
4 * error_bundle.extra.len + error_bundle.string_bytes.len;
try s.serveMessage(.{
.tag = .error_bundle,
.bytes_len = @as(u32, @intCast(bytes_len)),
.bytes_len = @intCast(bytes_len),
}, &.{
std.mem.asBytes(&eb_hdr),
// TODO: implement @ptrCast between slices changing the length
@ -251,7 +263,7 @@ pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void {
return s.serveMessage(.{
.tag = .test_metadata,
.bytes_len = @as(u32, @intCast(bytes_len)),
.bytes_len = @intCast(bytes_len),
}, &.{
std.mem.asBytes(&header),
// TODO: implement @ptrCast between slices changing the length

45
lib/std/zig/tokenizer.zig
View file

@ -1840,3 +1840,48 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v
try std.testing.expectEqual(source.len, last_token.loc.start);
try std.testing.expectEqual(source.len, last_token.loc.end);
}
test "fuzzable properties upheld" {
const source = std.testing.fuzzInput(.{});
const source0 = try std.testing.allocator.dupeZ(u8, source);
defer std.testing.allocator.free(source0);
var tokenizer = Tokenizer.init(source0);
var tokenization_failed = false;
while (true) {
const token = tokenizer.next();
// Property: token end location after start location (or equal)
try std.testing.expect(token.loc.end >= token.loc.start);
switch (token.tag) {
.invalid => {
tokenization_failed = true;
// Property: invalid token always ends at newline or eof
try std.testing.expect(source0[token.loc.end] == '\n' or source0[token.loc.end] == 0);
},
.eof => {
// Property: EOF token is always 0-length at end of source.
try std.testing.expectEqual(source0.len, token.loc.start);
try std.testing.expectEqual(source0.len, token.loc.end);
break;
},
else => continue,
}
}
if (source0.len > 0) for (source0, source0[1..][0..source0.len]) |cur, next| {
// Property: No null byte allowed except at end.
if (cur == 0) {
try std.testing.expect(tokenization_failed);
}
// Property: No ASCII control characters other than \n and \t are allowed.
if (std.ascii.isControl(cur) and cur != '\n' and cur != '\t') {
try std.testing.expect(tokenization_failed);
}
// Property: All '\r' must be followed by '\n'.
if (cur == '\r' and next != '\n') {
try std.testing.expect(tokenization_failed);
}
};
}