integrate the build runner and the compiler server

The compiler now provides a server protocol for an interactive session
with another process. The build runner uses this protocol to communicate
compilation errors semantically from zig compiler subprocesses to the
build runner.

The protocol is exposed via stdin/stdout, or on a network socket,
depending on whether the CLI flag `--listen=-` or e.g.
`--listen=127.0.0.1:1337` is used.

Additionally:

 * add the zig version string to the build runner cache prefix

 * remove --prominent-compile-errors CLI flag because it no longer does
   anything. Compilation errors are now unconditionally displayed at the
   bottom of the build summary output when using the terminal-based
   build runner.

 * Remove the color field from std.Build. The build steps are no longer
   supposed to interact with stderr directly. Instead they communicate
   semantically back to the build runner, which has its own logic about
   TTY configuration.

 * Use the cleanExit() pattern in the build runner.

 * Build steps can now use error.MakeFailed when they have already
   properly reported an error, or they can fail with any other error
   code in which case the build runner will create a simple message
   based on this error code.

lib/build_runner.zig

@@ -1,6 +1,7 @@
 const root = @import("@build");
 const std = @import("std");
 const builtin = @import("builtin");
+const assert = std.debug.assert;
 const io = std.io;
 const fmt = std.fmt;
 const mem = std.mem;
@@ -71,8 +72,7 @@ pub fn main() !void {
     cache.addPrefix(build_root_directory);
     cache.addPrefix(local_cache_directory);
     cache.addPrefix(global_cache_directory);
-
+    cache.hash.addBytes(builtin.zig_version_string);
-    //cache.hash.addBytes(builtin.zig_version);
 
     const builder = try std.Build.create(
         allocator,
@@ -95,10 +95,8 @@ pub fn main() !void {
     var install_prefix: ?[]const u8 = null;
     var dir_list = std.Build.DirList{};
 
-    // before arg parsing, check for the NO_COLOR environment variable
+    const Color = enum { auto, off, on };
-    // if it exists, default the color setting to .off
+    var color: Color = .auto;
-    // explicit --color arguments will still override this setting.
-    builder.color = if (process.hasEnvVarConstant("NO_COLOR")) .off else .auto;
 
     while (nextArg(args, &arg_idx)) |arg| {
         if (mem.startsWith(u8, arg, "-D")) {
@@ -166,7 +164,7 @@ pub fn main() !void {
                     std.debug.print("expected [auto|on|off] after --color", .{});
                     usageAndErr(builder, false, stderr_stream);
                 };
-                builder.color = std.meta.stringToEnum(@TypeOf(builder.color), next_arg) orelse {
+                color = std.meta.stringToEnum(Color, next_arg) orelse {
                     std.debug.print("expected [auto|on|off] after --color, found '{s}'", .{next_arg});
                     usageAndErr(builder, false, stderr_stream);
                 };
@@ -200,8 +198,6 @@ pub fn main() !void {
                 builder.verbose_cc = true;
             } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
                 builder.verbose_llvm_cpu_features = true;
-            } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
-                builder.prominent_compile_errors = true;
             } else if (mem.eql(u8, arg, "-fwine")) {
                 builder.enable_wine = true;
             } else if (mem.eql(u8, arg, "-fno-wine")) {
@@ -257,6 +253,12 @@ pub fn main() !void {
         }
     }
 
+    const ttyconf: std.debug.TTY.Config = switch (color) {
+        .auto => std.debug.detectTTYConfig(std.io.getStdErr()),
+        .on => .escape_codes,
+        .off => .no_color,
+    };
+
     var progress: std.Progress = .{};
     const main_progress_node = progress.start("", 0);
     defer main_progress_node.end();
@@ -272,11 +274,15 @@ pub fn main() !void {
     if (builder.validateUserInputDidItFail())
         usageAndErr(builder, true, stderr_stream);
 
-    runStepNames(builder, targets.items, main_progress_node, thread_pool_options) catch |err| {
+    runStepNames(
-        switch (err) {
+        builder,
-            error.UncleanExit => process.exit(1),
+        targets.items,
-            else => return err,
+        main_progress_node,
-        }
+        thread_pool_options,
+        ttyconf,
+    ) catch |err| switch (err) {
+        error.UncleanExit => process.exit(1),
+        else => return err,
     };
 }
 
@@ -285,6 +291,7 @@ fn runStepNames(
     step_names: []const []const u8,
     parent_prog_node: *std.Progress.Node,
     thread_pool_options: std.Thread.Pool.Options,
+    ttyconf: std.debug.TTY.Config,
 ) !void {
     var step_stack = ArrayList(*Step).init(b.allocator);
     defer step_stack.deinit();
@@ -332,12 +339,14 @@ fn runStepNames(
 
             wait_group.start();
             thread_pool.spawn(workerMakeOneStep, .{
-                &wait_group, &thread_pool, b, step, &step_prog,
+                &wait_group, &thread_pool, b, step, &step_prog, ttyconf,
             }) catch @panic("OOM");
         }
     }
 
-    var any_failed = false;
+    var success_count: usize = 0;
+    var failure_count: usize = 0;
+    var pending_count: usize = 0;
 
     for (step_stack.items) |s| {
         switch (s.state) {
@@ -349,20 +358,42 @@ fn runStepNames(
             // A -> B -> C (failure)
             // B will be marked as dependency_failure, while A may never be queued, and thus
             // remain in the initial state of precheck_done.
-            .dependency_failure, .precheck_done => continue,
+            .dependency_failure, .precheck_done => pending_count += 1,
-            .success => continue,
+            .success => success_count += 1,
-            .failure => {
+            .failure => failure_count += 1,
-                any_failed = true;
-                std.debug.print("{s}: {s}\n", .{
-                    s.name, @errorName(s.result.err_code),
-                });
-            },
         }
     }
 
-    if (any_failed) {
+    const stderr = std.io.getStdErr();
-        process.exit(1);
+
-    }
+    const total_count = success_count + failure_count + pending_count;
+    stderr.writer().print("build summary: {d}/{d} steps succeeded; {d} failed\n", .{
+        success_count, total_count, failure_count,
+    }) catch {};
+    if (failure_count == 0) return cleanExit();
+
+    for (step_stack.items) |s| switch (s.state) {
+        .failure => {
+            // TODO print the dep prefix too
+            ttyconf.setColor(stderr, .Bold) catch break;
+            stderr.writeAll(s.name) catch break;
+            ttyconf.setColor(stderr, .Reset) catch break;
+
+            if (s.result_error_bundle.errorMessageCount() > 0) {
+                stderr.writer().print(": {d} compilation errors:\n", .{
+                    s.result_error_bundle.errorMessageCount(),
+                }) catch break;
+                s.result_error_bundle.renderToStdErr(ttyconf);
+            } else {
+                stderr.writer().print(": {d} error messages (printed above)\n", .{
+                    s.result_error_msgs.items.len,
+                }) catch break;
+            }
+        },
+        else => continue,
+    };
+
+    process.exit(1);
 }
 
 fn checkForDependencyLoop(
@@ -407,6 +438,7 @@ fn workerMakeOneStep(
     b: *std.Build,
     s: *Step,
     prog_node: *std.Progress.Node,
+    ttyconf: std.debug.TTY.Config,
 ) void {
     defer wg.finish();
 
@@ -446,17 +478,26 @@ fn workerMakeOneStep(
     const make_result = s.make();
 
     // No matter the result, we want to display error/warning messages.
-    if (s.result.error_msgs.items.len > 0) {
+    if (s.result_error_msgs.items.len > 0) {
         sub_prog_node.context.lock_stderr();
         defer sub_prog_node.context.unlock_stderr();
 
-        for (s.result.error_msgs.items) |msg| {
+        const stderr = std.io.getStdErr();
-            std.io.getStdErr().writeAll(msg) catch break;
+
+        for (s.result_error_msgs.items) |msg| {
+            // TODO print the dep prefix too
+            ttyconf.setColor(stderr, .Bold) catch break;
+            stderr.writeAll(s.name) catch break;
+            stderr.writeAll(": ") catch break;
+            ttyconf.setColor(stderr, .Red) catch break;
+            stderr.writeAll("error: ") catch break;
+            ttyconf.setColor(stderr, .Reset) catch break;
+            stderr.writeAll(msg) catch break;
         }
     }
 
     make_result catch |err| {
-        s.result.err_code = err;
+        assert(err == error.MakeFailed);
         @atomicStore(Step.State, &s.state, .failure, .SeqCst);
         return;
     };
@@ -467,7 +508,7 @@ fn workerMakeOneStep(
     for (s.dependants.items) |dep| {
         wg.start();
         thread_pool.spawn(workerMakeOneStep, .{
-            wg, thread_pool, b, dep, prog_node,
+            wg, thread_pool, b, dep, prog_node, ttyconf,
         }) catch @panic("OOM");
     }
 }
@@ -601,3 +642,11 @@ fn argsRest(args: [][]const u8, idx: usize) ?[][]const u8 {
     if (idx >= args.len) return null;
     return args[idx..];
 }
+
+fn cleanExit() void {
+    if (builtin.mode == .Debug) {
+        return;
+    } else {
+        process.exit(0);
+    }
+}

lib/std/Build.zig

@@ -59,9 +59,6 @@ verbose_air: bool,
 verbose_llvm_ir: bool,
 verbose_cimport: bool,
 verbose_llvm_cpu_features: bool,
-/// The purpose of executing the command is for a human to read compile errors from the terminal
-prominent_compile_errors: bool,
-color: enum { auto, on, off } = .auto,
 reference_trace: ?u32 = null,
 invalid_user_input: bool,
 zig_exe: []const u8,
@@ -211,7 +208,6 @@ pub fn create(
         .verbose_llvm_ir = false,
         .verbose_cimport = false,
         .verbose_llvm_cpu_features = false,
-        .prominent_compile_errors = false,
         .invalid_user_input = false,
         .allocator = allocator,
         .user_input_options = UserInputOptionsMap.init(allocator),
@@ -295,8 +291,6 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
         .verbose_llvm_ir = parent.verbose_llvm_ir,
         .verbose_cimport = parent.verbose_cimport,
         .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features,
-        .prominent_compile_errors = parent.prominent_compile_errors,
-        .color = parent.color,
         .reference_trace = parent.reference_trace,
         .invalid_user_input = false,
         .zig_exe = parent.zig_exe,
@@ -1409,54 +1403,149 @@ pub fn execAllowFail(
     }
 }
 
-pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]u8 {
+/// This function is used exclusively for spawning and communicating with the zig compiler.
+/// TODO: move to build_runner.zig
+pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
     assert(argv.len != 0);
 
     if (b.verbose) {
-        printCmd(b.allocator, null, argv);
+        const text = try allocPrintCmd(b.allocator, null, argv);
+        try s.result_error_msgs.append(b.allocator, text);
     }
 
     if (!process.can_spawn) {
-        try s.result.error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{
+        try s.result_error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{
             try allocPrintCmd(b.allocator, null, argv),
         }));
-        return error.CannotSpawnProcesses;
+        return error.MakeFailed;
     }
 
-    const result = std.ChildProcess.exec(.{
+    var child = std.ChildProcess.init(argv, b.allocator);
-        .allocator = b.allocator,
+    child.env_map = b.env_map;
-        .argv = argv,
+    child.stdin_behavior = .Pipe;
-        .env_map = b.env_map,
+    child.stdout_behavior = .Pipe;
-        .max_output_bytes = 10 * 1024 * 1024,
+    child.stderr_behavior = .Pipe;
-    }) catch |err| {
-        try s.result.error_msgs.append(b.allocator, b.fmt("unable to spawn the following command: {s}\n{s}", .{
-            @errorName(err), try allocPrintCmd(b.allocator, null, argv),
-        }));
-        return error.ExecFailed;
-    };
 
-    if (result.stderr.len != 0) {
+    try child.spawn();
-        try s.result.error_msgs.append(b.allocator, result.stderr);
+
+    var poller = std.io.poll(b.allocator, enum { stdout, stderr }, .{
+        .stdout = child.stdout.?,
+        .stderr = child.stderr.?,
+    });
+    defer poller.deinit();
+
+    try sendMessage(child.stdin.?, .update);
+    try sendMessage(child.stdin.?, .exit);
+
+    const Header = std.zig.Server.Message.Header;
+    var result: ?[]const u8 = null;
+
+    while (try poller.poll()) {
+        const stdout = poller.fifo(.stdout);
+        const buf = stdout.readableSlice(0);
+        assert(stdout.readableLength() == buf.len);
+        if (buf.len >= @sizeOf(Header)) {
+            const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
+            const header_and_msg_len = header.bytes_len + @sizeOf(Header);
+            if (buf.len >= header_and_msg_len) {
+                const body = buf[@sizeOf(Header)..];
+                switch (header.tag) {
+                    .zig_version => {
+                        if (!mem.eql(u8, builtin.zig_version_string, body)) {
+                            try s.result_error_msgs.append(
+                                b.allocator,
+                                b.fmt("zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{
+                                    builtin.zig_version_string, body,
+                                }),
+                            );
+                            return error.MakeFailed;
+                        }
+                    },
+                    .error_bundle => {
+                        const EbHdr = std.zig.Server.Message.ErrorBundle;
+                        const eb_hdr = @ptrCast(*align(1) const EbHdr, body);
+                        const extra_bytes =
+                            body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
+                        const string_bytes =
+                            body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
+                        // TODO: use @ptrCast when the compiler supports it
+                        const unaligned_extra = mem.bytesAsSlice(u32, extra_bytes);
+                        const extra_array = try b.allocator.alloc(u32, unaligned_extra.len);
+                        // TODO: use @memcpy when it supports slices
+                        for (extra_array, unaligned_extra) |*dst, src| dst.* = src;
+                        s.result_error_bundle = .{
+                            .string_bytes = try b.allocator.dupe(u8, string_bytes),
+                            .extra = extra_array,
+                        };
+                    },
+                    .progress => {
+                        @panic("TODO handle progress message");
+                    },
+                    .emit_bin_path => {
+                        @panic("TODO handle emit_bin_path message");
+                    },
+                    _ => {
+                        // Unrecognized message.
+                    },
+                }
+                stdout.discard(header_and_msg_len);
+            }
+        }
     }
 
-    switch (result.term) {
+    const stderr = poller.fifo(.stderr);
+    if (stderr.readableLength() > 0) {
+        try s.result_error_msgs.append(b.allocator, try stderr.toOwnedSlice());
+    }
+
+    // Send EOF to stdin.
+    child.stdin.?.close();
+    child.stdin = null;
+
+    const term = try child.wait();
+    switch (term) {
         .Exited => |code| {
             if (code != 0) {
-                try s.result.error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{
+                try s.result_error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{
                     code, try allocPrintCmd(b.allocator, null, argv),
                 }));
-                return error.ExitCodeFailure;
+                return error.MakeFailed;
             }
-            return result.stdout;
         },
         .Signal, .Stopped, .Unknown => |code| {
             _ = code;
-            try s.result.error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{
+            try s.result_error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{
                 try allocPrintCmd(b.allocator, null, argv),
             }));
-            return error.ProcessTerminated;
+            return error.MakeFailed;
         },
     }
+
+    if (s.result_error_bundle.errorMessageCount() > 0) {
+        try s.result_error_msgs.append(
+            b.allocator,
+            b.fmt("the following command failed with {d} compilation errors:\n{s}", .{
+                s.result_error_bundle.errorMessageCount(),
+                try allocPrintCmd(b.allocator, null, argv),
+            }),
+        );
+        return error.MakeFailed;
+    }
+
+    return result orelse {
+        try s.result_error_msgs.append(b.allocator, b.fmt("the following command failed to communicate the compilation result:\n{s}", .{
+            try allocPrintCmd(b.allocator, null, argv),
+        }));
+        return error.MakeFailed;
+    };
+}
+
+fn sendMessage(file: fs.File, tag: std.zig.Client.Message.Tag) !void {
+    const header: std.zig.Client.Message.Header = .{
+        .tag = tag,
+        .bytes_len = 0,
+    };
+    try file.writeAll(std.mem.asBytes(&header));
 }
 
 /// This is a helper function to be called from build.zig scripts, *not* from

lib/std/Build/CompileStep.zig

@@ -1177,11 +1177,6 @@ fn make(step: *Step) !void {
     };
     try zig_args.append(cmd);
 
-    if (builder.color != .auto) {
-        try zig_args.append("--color");
-        try zig_args.append(@tagName(builder.color));
-    }
-
     if (builder.reference_trace) |some| {
         try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-freference-trace={d}", .{some}));
     }
@@ -1834,6 +1829,7 @@ fn make(step: *Step) !void {
     }
 
     try zig_args.append("--enable-cache");
+    try zig_args.append("--listen=-");
 
     // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
     // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and

lib/std/Build/RunStep.zig

@@ -419,12 +419,8 @@ pub fn runCommand(
     };
 
     if (!termMatches(expected_term, term)) {
-        if (builder.prominent_compile_errors) {
+        std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) });
-            std.debug.print("Run step {} (expected {})\n", .{ fmtTerm(term), fmtTerm(expected_term) });
+        printCmd(cwd, argv);
-        } else {
-            std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) });
-            printCmd(cwd, argv);
-        }
         return error.UnexpectedExit;
     }
 

lib/std/Build/Step.zig

@@ -6,15 +6,13 @@ dependencies: std.ArrayList(*Step),
 /// then populated during dependency loop checking in the build runner.
 dependants: std.ArrayListUnmanaged(*Step),
 state: State,
-/// Populated only if state is success.
-result: struct {
-    err_code: anyerror,
-    error_msgs: std.ArrayListUnmanaged([]const u8),
-},
 /// The return addresss associated with creation of this step that can be useful
 /// to print along with debugging messages.
 debug_stack_trace: [n_debug_stack_frames]usize,
 
+result_error_msgs: std.ArrayListUnmanaged([]const u8),
+result_error_bundle: std.zig.ErrorBundle,
+
 const n_debug_stack_frames = 4;
 
 pub const State = enum {
@@ -94,16 +92,25 @@ pub fn init(allocator: Allocator, options: Options) Step {
         .dependencies = std.ArrayList(*Step).init(allocator),
         .dependants = .{},
         .state = .precheck_unstarted,
-        .result = .{
-            .err_code = undefined,
-            .error_msgs = .{},
-        },
         .debug_stack_trace = addresses,
+        .result_error_msgs = .{},
+        .result_error_bundle = std.zig.ErrorBundle.empty,
     };
 }
 
-pub fn make(self: *Step) !void {
+/// If the Step's `make` function reports `error.MakeFailed`, it indicates they
-    try self.makeFn(self);
+/// have already reported the error. Otherwise, we add a simple error report
+/// here.
+pub fn make(s: *Step) error{MakeFailed}!void {
+    return s.makeFn(s) catch |err| {
+        if (err != error.MakeFailed) {
+            const gpa = s.dependencies.allocator;
+            s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{
+                s.name, @errorName(err),
+            }) catch @panic("OOM")) catch @panic("OOM");
+        }
+        return error.MakeFailed;
+    };
 }
 
 pub fn dependOn(self: *Step, other: *Step) void {

lib/std/zig.zig

@@ -4,6 +4,8 @@ const fmt = @import("zig/fmt.zig");
 const assert = std.debug.assert;
 
 pub const ErrorBundle = @import("zig/ErrorBundle.zig");
+pub const Server = @import("zig/Server.zig");
+pub const Client = @import("zig/Client.zig");
 pub const Token = tokenizer.Token;
 pub const Tokenizer = tokenizer.Tokenizer;
 pub const fmtId = fmt.fmtId;

lib/std/zig/Client.zig

@@ -0,0 +1,32 @@
+pub const Message = struct {
+    pub const Header = extern struct {
+        tag: Tag,
+        /// Size of the body only; does not include this Header.
+        bytes_len: u32,
+    };
+
+    pub const Tag = enum(u32) {
+        /// Tells the compiler to shut down cleanly.
+        /// No body.
+        exit,
+        /// Tells the compiler to detect changes in source files and update the
+        /// affected output compilation artifacts.
+        /// If one of the compilation artifacts is an executable that is
+        /// running as a child process, the compiler will wait for it to exit
+        /// before performing the update.
+        /// No body.
+        update,
+        /// Tells the compiler to execute the executable as a child process.
+        /// No body.
+        run,
+        /// Tells the compiler to detect changes in source files and update the
+        /// affected output compilation artifacts.
+        /// If one of the compilation artifacts is an executable that is
+        /// running as a child process, the compiler will perform a hot code
+        /// swap.
+        /// No body.
+        hot_update,
+
+        _,
+    };
+};

lib/std/zig/Server.zig

@@ -0,0 +1,28 @@
+pub const Message = struct {
+    pub const Header = extern struct {
+        tag: Tag,
+        /// Size of the body only; does not include this Header.
+        bytes_len: u32,
+    };
+
+    pub const Tag = enum(u32) {
+        /// Body is a UTF-8 string.
+        zig_version,
+        /// Body is an ErrorBundle.
+        error_bundle,
+        /// Body is a UTF-8 string.
+        progress,
+        /// Body is a UTF-8 string.
+        emit_bin_path,
+        _,
+    };
+
+    /// Trailing:
+    /// * extra: [extra_len]u32,
+    /// * string_bytes: [string_bytes_len]u8,
+    /// See `std.zig.ErrorBundle`.
+    pub const ErrorBundle = extern struct {
+        extra_len: u32,
+        string_bytes_len: u32,
+    };
+};

src/main.zig

@@ -668,6 +668,12 @@ const ArgMode = union(enum) {
     run,
 };
 
+const Listen = union(enum) {
+    none,
+    ip4: std.net.Ip4Address,
+    stdio,
+};
+
 fn buildOutputType(
     gpa: Allocator,
     arena: Allocator,
@@ -689,7 +695,7 @@ fn buildOutputType(
     var function_sections = false;
     var no_builtin = false;
     var watch = false;
-    var listen_addr: ?std.net.Ip4Address = null;
+    var listen: Listen = .none;
     var debug_compile_errors = false;
     var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK");
     var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_CC");
@@ -1149,14 +1155,22 @@ fn buildOutputType(
                         }
                     } else if (mem.eql(u8, arg, "--listen")) {
                         const next_arg = args_iter.nextOrFatal();
-                        // example: --listen 127.0.0.1:9000
+                        if (mem.eql(u8, next_arg, "-")) {
-                        var it = std.mem.split(u8, next_arg, ":");
+                            listen = .stdio;
-                        const host = it.next().?;
+                            watch = true;
-                        const port_text = it.next() orelse "14735";
+                        } else {
-                        const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
+                            // example: --listen 127.0.0.1:9000
-                            fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) });
+                            var it = std.mem.split(u8, next_arg, ":");
-                        listen_addr = std.net.Ip4Address.parse(host, port) catch |err|
+                            const host = it.next().?;
-                            fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) });
+                            const port_text = it.next() orelse "14735";
+                            const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
+                                fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) });
+                            listen = .{ .ip4 = std.net.Ip4Address.parse(host, port) catch |err|
+                                fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }) };
+                            watch = true;
+                        }
+                    } else if (mem.eql(u8, arg, "--listen=-")) {
+                        listen = .stdio;
                         watch = true;
                     } else if (mem.eql(u8, arg, "--debug-link-snapshot")) {
                         if (!build_options.enable_link_snapshots) {
@@ -3277,6 +3291,47 @@ fn buildOutputType(
         return cmdTranslateC(comp, arena, have_enable_cache);
     }
 
+    switch (listen) {
+        .none => {},
+        .stdio => {
+            try serve(
+                comp,
+                std.io.getStdIn(),
+                std.io.getStdOut(),
+                test_exec_args.items,
+                self_exe_path,
+                arg_mode,
+                all_args,
+                runtime_args_start,
+            );
+            return cleanExit();
+        },
+        .ip4 => |ip4_addr| {
+            var server = std.net.StreamServer.init(.{
+                .reuse_address = true,
+            });
+            defer server.deinit();
+
+            try server.listen(.{ .in = ip4_addr });
+
+            while (true) {
+                const conn = try server.accept();
+                defer conn.stream.close();
+
+                try serve(
+                    comp,
+                    .{ .handle = conn.stream.handle },
+                    .{ .handle = conn.stream.handle },
+                    test_exec_args.items,
+                    self_exe_path,
+                    arg_mode,
+                    all_args,
+                    runtime_args_start,
+                );
+            }
+        },
+    }
+
     const hook: AfterUpdateHook = blk: {
         if (!have_enable_cache)
             break :blk .none;
@@ -3354,6 +3409,12 @@ fn buildOutputType(
         );
     }
 
+    // TODO move this REPL implementation to the standard library / build
+    // system and have it be a CLI abstraction layer on top of the real, actual
+    // binary protocol of the compiler. Make it actually interface through the
+    // server protocol. This way the REPL does not have any special powers that
+    // an IDE couldn't also have.
+
     const stdin = std.io.getStdIn().reader();
     const stderr = std.io.getStdErr().writer();
     var repl_buf: [1024]u8 = undefined;
@@ -3367,123 +3428,6 @@ fn buildOutputType(
 
     var last_cmd: ReplCmd = .help;
 
-    if (listen_addr) |ip4_addr| {
-        var server = std.net.StreamServer.init(.{
-            .reuse_address = true,
-        });
-        defer server.deinit();
-
-        try server.listen(.{ .in = ip4_addr });
-
-        while (true) {
-            const conn = try server.accept();
-            defer conn.stream.close();
-
-            var buf: [100]u8 = undefined;
-            var child_pid: ?i32 = null;
-
-            while (true) {
-                try comp.makeBinFileExecutable();
-
-                const amt = try conn.stream.read(&buf);
-                const line = buf[0..amt];
-                const actual_line = mem.trimRight(u8, line, "\r\n ");
-
-                const cmd: ReplCmd = blk: {
-                    if (mem.eql(u8, actual_line, "update")) {
-                        break :blk .update;
-                    } else if (mem.eql(u8, actual_line, "exit")) {
-                        break;
-                    } else if (mem.eql(u8, actual_line, "help")) {
-                        break :blk .help;
-                    } else if (mem.eql(u8, actual_line, "run")) {
-                        break :blk .run;
-                    } else if (mem.eql(u8, actual_line, "update-and-run")) {
-                        break :blk .update_and_run;
-                    } else if (actual_line.len == 0) {
-                        break :blk last_cmd;
-                    } else {
-                        try stderr.print("unknown command: {s}\n", .{actual_line});
-                        continue;
-                    }
-                };
-                last_cmd = cmd;
-                switch (cmd) {
-                    .update => {
-                        tracy.frameMark();
-                        if (output_mode == .Exe) {
-                            try comp.makeBinFileWritable();
-                        }
-                        updateModule(gpa, comp, hook) catch |err| switch (err) {
-                            error.SemanticAnalyzeFail => continue,
-                            else => |e| return e,
-                        };
-                    },
-                    .help => {
-                        try stderr.writeAll(repl_help);
-                    },
-                    .run => {
-                        tracy.frameMark();
-                        try runOrTest(
-                            comp,
-                            gpa,
-                            arena,
-                            test_exec_args.items,
-                            self_exe_path.?,
-                            arg_mode,
-                            target_info,
-                            watch,
-                            &comp_destroyed,
-                            all_args,
-                            runtime_args_start,
-                            link_libc,
-                        );
-                    },
-                    .update_and_run => {
-                        tracy.frameMark();
-                        if (child_pid) |pid| {
-                            try conn.stream.writer().print("hot code swap requested for pid {d}", .{pid});
-                            try comp.hotCodeSwap(pid);
-
-                            var errors = try comp.getAllErrorsAlloc();
-                            defer errors.deinit(comp.gpa);
-
-                            if (errors.errorMessageCount() > 0) {
-                                const ttyconf: std.debug.TTY.Config = switch (comp.color) {
-                                    .auto => std.debug.detectTTYConfig(std.io.getStdErr()),
-                                    .on => .escape_codes,
-                                    .off => .no_color,
-                                };
-                                try errors.renderToWriter(ttyconf, conn.stream.writer());
-                                continue;
-                            }
-                        } else {
-                            if (output_mode == .Exe) {
-                                try comp.makeBinFileWritable();
-                            }
-                            updateModule(gpa, comp, hook) catch |err| switch (err) {
-                                error.SemanticAnalyzeFail => continue,
-                                else => |e| return e,
-                            };
-                            try comp.makeBinFileExecutable();
-
-                            child_pid = try runOrTestHotSwap(
-                                comp,
-                                gpa,
-                                arena,
-                                test_exec_args.items,
-                                self_exe_path.?,
-                                arg_mode,
-                                all_args,
-                                runtime_args_start,
-                            );
-                        }
-                    },
-                }
-            }
-        }
-    }
-
     while (watch) {
         try stderr.print("(zig) ", .{});
         try comp.makeBinFileExecutable();
@@ -3576,6 +3520,173 @@ fn buildOutputType(
     return cleanExit();
 }
 
+fn serve(
+    comp: *Compilation,
+    in: fs.File,
+    out: fs.File,
+    test_exec_args: []const ?[]const u8,
+    self_exe_path: ?[]const u8,
+    arg_mode: ArgMode,
+    all_args: []const []const u8,
+    runtime_args_start: ?usize,
+) !void {
+    const gpa = comp.gpa;
+
+    try serveMessage(out, .{
+        .tag = .zig_version,
+        .bytes_len = build_options.version.len,
+    }, &.{
+        build_options.version,
+    });
+
+    var child_pid: ?i32 = null;
+    var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa);
+    defer receive_fifo.deinit();
+
+    while (true) {
+        const hdr = try receiveMessage(in, &receive_fifo);
+
+        switch (hdr.tag) {
+            .exit => {
+                return cleanExit();
+            },
+            .update => {
+                tracy.frameMark();
+                if (comp.bin_file.options.output_mode == .Exe) {
+                    try comp.makeBinFileWritable();
+                }
+                try comp.update();
+                try comp.makeBinFileExecutable();
+                try serveUpdateResults(out, comp);
+            },
+            .run => {
+                if (child_pid != null) {
+                    @panic("TODO block until the child exits");
+                }
+                @panic("TODO call runOrTest");
+                //try runOrTest(
+                //    comp,
+                //    gpa,
+                //    arena,
+                //    test_exec_args,
+                //    self_exe_path.?,
+                //    arg_mode,
+                //    target_info,
+                //    true,
+                //    &comp_destroyed,
+                //    all_args,
+                //    runtime_args_start,
+                //    link_libc,
+                //);
+            },
+            .hot_update => {
+                tracy.frameMark();
+                if (child_pid) |pid| {
+                    try comp.hotCodeSwap(pid);
+                    try serveUpdateResults(out, comp);
+                } else {
+                    if (comp.bin_file.options.output_mode == .Exe) {
+                        try comp.makeBinFileWritable();
+                    }
+                    try comp.update();
+                    try comp.makeBinFileExecutable();
+                    try serveUpdateResults(out, comp);
+
+                    child_pid = try runOrTestHotSwap(
+                        comp,
+                        gpa,
+                        test_exec_args,
+                        self_exe_path.?,
+                        arg_mode,
+                        all_args,
+                        runtime_args_start,
+                    );
+                }
+            },
+            _ => {
+                @panic("TODO unrecognized message from client");
+            },
+        }
+    }
+}
+
+fn serveMessage(
+    out: fs.File,
+    header: std.zig.Server.Message.Header,
+    bufs: []const []const u8,
+) !void {
+    var iovecs: [10]std.os.iovec_const = undefined;
+    iovecs[0] = .{
+        .iov_base = @ptrCast([*]const u8, &header),
+        .iov_len = @sizeOf(std.zig.Server.Message.Header),
+    };
+    for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
+        iovec.* = .{
+            .iov_base = buf.ptr,
+            .iov_len = buf.len,
+        };
+    }
+    try out.writevAll(iovecs[0 .. bufs.len + 1]);
+}
+
+fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void {
+    const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
+        .extra_len = @intCast(u32, error_bundle.extra.len),
+        .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
+    };
+    const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
+        4 * error_bundle.extra.len + error_bundle.string_bytes.len;
+    try serveMessage(out, .{
+        .tag = .error_bundle,
+        .bytes_len = @intCast(u32, bytes_len),
+    }, &.{
+        std.mem.asBytes(&eb_hdr),
+        // TODO: implement @ptrCast between slices changing the length
+        std.mem.sliceAsBytes(error_bundle.extra),
+        error_bundle.string_bytes,
+    });
+}
+
+fn serveUpdateResults(out: fs.File, comp: *Compilation) !void {
+    const gpa = comp.gpa;
+    var error_bundle = try comp.getAllErrorsAlloc();
+    defer error_bundle.deinit(gpa);
+    if (error_bundle.errorMessageCount() > 0) {
+        try serveErrorBundle(out, error_bundle);
+    } else if (comp.bin_file.options.emit) |emit| {
+        const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
+        defer gpa.free(full_path);
+
+        try serveMessage(out, .{
+            .tag = .emit_bin_path,
+            .bytes_len = @intCast(u32, full_path.len),
+        }, &.{
+            full_path,
+        });
+    }
+}
+
+fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header {
+    const Header = std.zig.Client.Message.Header;
+
+    while (true) {
+        const buf = fifo.readableSlice(0);
+        assert(fifo.readableLength() == buf.len);
+        if (buf.len >= @sizeOf(Header)) {
+            const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
+            if (header.bytes_len != 0)
+                return error.InvalidClientMessage;
+            const result = header.*;
+            fifo.discard(@sizeOf(Header));
+            return result;
+        }
+
+        const write_buffer = try fifo.writableWithSize(256);
+        const amt = try in.read(write_buffer);
+        fifo.update(amt);
+    }
+}
+
 const ModuleDepIterator = struct {
     split: mem.SplitIterator(u8),
 
@@ -3765,7 +3876,6 @@ fn runOrTest(
 fn runOrTestHotSwap(
     comp: *Compilation,
     gpa: Allocator,
-    arena: Allocator,
     test_exec_args: []const ?[]const u8,
     self_exe_path: []const u8,
     arg_mode: ArgMode,
@@ -3775,9 +3885,10 @@ fn runOrTestHotSwap(
     const exe_emit = comp.bin_file.options.emit.?;
     // A naive `directory.join` here will indeed get the correct path to the binary,
     // however, in the case of cwd, we actually want `./foo` so that the path can be executed.
-    const exe_path = try fs.path.join(arena, &[_][]const u8{
+    const exe_path = try fs.path.join(gpa, &[_][]const u8{
         exe_emit.directory.path orelse ".", exe_emit.sub_path,
     });
+    defer gpa.free(exe_path);
 
     var argv = std.ArrayList([]const u8).init(gpa);
     defer argv.deinit();
@@ -3807,7 +3918,7 @@ fn runOrTestHotSwap(
     if (runtime_args_start) |i| {
         try argv.appendSlice(all_args[i..]);
     }
-    var child = std.ChildProcess.init(argv.items, arena);
+    var child = std.ChildProcess.init(argv.items, gpa);
 
     child.stdin_behavior = .Inherit;
     child.stdout_behavior = .Inherit;
@@ -4206,7 +4317,6 @@ pub const usage_build =
 
 pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
     var color: Color = .auto;
-    var prominent_compile_errors: bool = false;
 
     // We want to release all the locks before executing the child process, so we make a nice
     // big block here to ensure the cleanup gets run when we extract out our argv.
@@ -4267,8 +4377,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
                         i += 1;
                         override_global_cache_dir = args[i];
                         continue;
-                    } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
-                        prominent_compile_errors = true;
                     } else if (mem.eql(u8, arg, "-freference-trace")) {
                         try child_argv.append(arg);
                         reference_trace = 256;
@@ -4535,12 +4643,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
             .Exited => |code| {
                 if (code == 0) return cleanExit();
 
-                if (prominent_compile_errors) {
+                const cmd = try std.mem.join(arena, " ", child_argv);
-                    fatal("the build command failed with exit code {d}", .{code});
+                fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
-                } else {
-                    const cmd = try std.mem.join(arena, " ", child_argv);
-                    fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
-                }
             },
             else => {
                 const cmd = try std.mem.join(arena, " ", child_argv);