zig std: implement serving the wasm binary

lib/compiler/std-docs.zig

@@ -15,9 +15,8 @@ pub fn main() !void {
     const zig_exe_path = args[2];
     const global_cache_path = args[3];
 
-    const docs_path = try std.fs.path.join(arena, &.{ zig_lib_directory, "docs" });
+    var lib_dir = try std.fs.cwd().openDir(zig_lib_directory, .{});
-    var docs_dir = try std.fs.cwd().openDir(docs_path, .{});
+    defer lib_dir.close();
-    defer docs_dir.close();
 
     const listen_port: u16 = 0;
     const address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable;
@@ -29,6 +28,14 @@ pub fn main() !void {
         std.log.err("unable to open browser: {s}", .{@errorName(err)});
     };
 
+    var context: Context = .{
+        .gpa = gpa,
+        .zig_exe_path = zig_exe_path,
+        .global_cache_path = global_cache_path,
+        .lib_dir = lib_dir,
+        .zig_lib_directory = zig_lib_directory,
+    };
+
     var read_buffer: [8000]u8 = undefined;
     accept: while (true) {
         const connection = try http_server.accept();
@@ -43,7 +50,7 @@ pub fn main() !void {
                     continue :accept;
                 },
             };
-            serveRequest(&request, gpa, docs_dir, zig_exe_path, global_cache_path) catch |err| {
+            serveRequest(&request, &context) catch |err| {
                 std.log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(err) });
                 continue :accept;
             };
@@ -51,25 +58,31 @@ pub fn main() !void {
     }
 }
 
-fn serveRequest(
+const Context = struct {
-    request: *std.http.Server.Request,
     gpa: Allocator,
-    docs_dir: std.fs.Dir,
+    lib_dir: std.fs.Dir,
+    zig_lib_directory: []const u8,
     zig_exe_path: []const u8,
     global_cache_path: []const u8,
-) !void {
+};
+
+fn serveRequest(request: *std.http.Server.Request, context: *Context) !void {
     if (std.mem.eql(u8, request.head.target, "/") or
         std.mem.eql(u8, request.head.target, "/debug/"))
     {
-        try serveDocsFile(request, gpa, docs_dir, "index.html", "text/html");
+        try serveDocsFile(request, context, "docs/index.html", "text/html");
     } else if (std.mem.eql(u8, request.head.target, "/main.js") or
         std.mem.eql(u8, request.head.target, "/debug/main.js"))
     {
-        try serveDocsFile(request, gpa, docs_dir, "main.js", "application/javascript");
+        try serveDocsFile(request, context, "docs/main.js", "application/javascript");
     } else if (std.mem.eql(u8, request.head.target, "/main.wasm")) {
-        try serveWasm(request, gpa, zig_exe_path, global_cache_path, .ReleaseFast);
+        try serveWasm(request, context, .ReleaseFast);
     } else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) {
-        try serveWasm(request, gpa, zig_exe_path, global_cache_path, .Debug);
+        try serveWasm(request, context, .Debug);
+    } else if (std.mem.eql(u8, request.head.target, "/sources.tar") or
+        std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
+    {
+        try serveSourcesTar(request, context);
     } else {
         try request.respond("not found", .{
             .status = .not_found,
@@ -80,68 +93,219 @@ fn serveRequest(
     }
 }
 
+const cache_control_header: std.http.Header = .{
+    .name = "cache-control",
+    .value = "max-age=0, must-revalidate",
+};
+
 fn serveDocsFile(
     request: *std.http.Server.Request,
-    gpa: Allocator,
+    context: *Context,
-    docs_dir: std.fs.Dir,
     name: []const u8,
     content_type: []const u8,
 ) !void {
+    const gpa = context.gpa;
     // The desired API is actually sendfile, which will require enhancing std.http.Server.
     // We load the file with every request so that the user can make changes to the file
     // and refresh the HTML page without restarting this server.
-    const file_contents = try docs_dir.readFileAlloc(gpa, name, 10 * 1024 * 1024);
+    const file_contents = try context.lib_dir.readFileAlloc(gpa, name, 10 * 1024 * 1024);
     defer gpa.free(file_contents);
     try request.respond(file_contents, .{
         .status = .ok,
         .extra_headers = &.{
             .{ .name = "content-type", .value = content_type },
+            cache_control_header,
         },
     });
 }
 
-fn serveWasm(
+fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void {
-    request: *std.http.Server.Request,
-    gpa: Allocator,
-    zig_exe_path: []const u8,
-    global_cache_path: []const u8,
-    optimize_mode: std.builtin.OptimizeMode,
-) !void {
     _ = request;
-    _ = gpa;
+    _ = context;
-    _ = zig_exe_path;
+    @panic("TODO");
-    _ = global_cache_path;
-    _ = optimize_mode;
-    @panic("TODO serve wasm");
 }
 
-const BuildWasmBinaryOptions = struct {
+fn serveWasm(
-    zig_exe_path: []const u8,
+    request: *std.http.Server.Request,
-    global_cache_path: []const u8,
+    context: *Context,
-    main_src_path: []const u8,
+    optimize_mode: std.builtin.OptimizeMode,
-};
+) !void {
+    const gpa = context.gpa;
+
+    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+    defer arena_instance.deinit();
+    const arena = arena_instance.allocator();
+
+    // Do the compilation every request, so that the user can edit the files
+    // and see the changes without restarting the server.
+    const wasm_binary_path = try buildWasmBinary(arena, context, optimize_mode);
+    // std.http.Server does not have a sendfile API yet.
+    const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
+    defer gpa.free(file_contents);
+    try request.respond(file_contents, .{
+        .status = .ok,
+        .extra_headers = &.{
+            .{ .name = "content-type", .value = "application/wasm" },
+            cache_control_header,
+        },
+    });
+}
+
+fn buildWasmBinary(
+    arena: Allocator,
+    context: *Context,
+    optimize_mode: std.builtin.OptimizeMode,
+) ![]const u8 {
+    const gpa = context.gpa;
+
+    const main_src_path = try std.fs.path.join(arena, &.{
+        context.zig_lib_directory, "docs", "wasm", "main.zig",
+    });
 
-fn buildWasmBinary(arena: Allocator, options: BuildWasmBinaryOptions) ![]const u8 {
     var argv: std.ArrayListUnmanaged([]const u8) = .{};
+
     try argv.appendSlice(arena, &.{
-        options.zig_exe_path,
+        context.zig_exe_path,
         "build-exe",
         "-fno-entry",
-        "-OReleaseSmall",
+        "-O",
+        @tagName(optimize_mode),
         "-target",
         "wasm32-freestanding",
         "-mcpu",
         "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",
         "--cache-dir",
-        options.global_cache_path,
+        context.global_cache_path,
         "--global-cache-dir",
-        options.global_cache_path,
+        context.global_cache_path,
         "--name",
         "autodoc",
         "-rdynamic",
-        options.main_src_path,
+        main_src_path,
         "--listen=-",
     });
+
+    var child = std.ChildProcess.init(argv.items, gpa);
+    child.stdin_behavior = .Pipe;
+    child.stdout_behavior = .Pipe;
+    child.stderr_behavior = .Pipe;
+    try child.spawn();
+
+    var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
+        .stdout = child.stdout.?,
+        .stderr = child.stderr.?,
+    });
+    defer poller.deinit();
+
+    try sendMessage(child.stdin.?, .update);
+    try sendMessage(child.stdin.?, .exit);
+
+    const Header = std.zig.Server.Message.Header;
+    var result: ?[]const u8 = null;
+    var result_error_bundle = std.zig.ErrorBundle.empty;
+
+    const stdout = poller.fifo(.stdout);
+
+    poll: while (true) {
+        while (stdout.readableLength() < @sizeOf(Header)) {
+            if (!(try poller.poll())) break :poll;
+        }
+        const header = stdout.reader().readStruct(Header) catch unreachable;
+        while (stdout.readableLength() < header.bytes_len) {
+            if (!(try poller.poll())) break :poll;
+        }
+        const body = stdout.readableSliceOfLen(header.bytes_len);
+
+        switch (header.tag) {
+            .zig_version => {
+                if (!std.mem.eql(u8, builtin.zig_version_string, body)) {
+                    return error.ZigProtocolVersionMismatch;
+                }
+            },
+            .error_bundle => {
+                const EbHdr = std.zig.Server.Message.ErrorBundle;
+                const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body));
+                const extra_bytes =
+                    body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
+                const string_bytes =
+                    body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
+                // TODO: use @ptrCast when the compiler supports it
+                const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes);
+                const extra_array = try arena.alloc(u32, unaligned_extra.len);
+                @memcpy(extra_array, unaligned_extra);
+                result_error_bundle = .{
+                    .string_bytes = try arena.dupe(u8, string_bytes),
+                    .extra = extra_array,
+                };
+            },
+            .emit_bin_path => {
+                const EbpHdr = std.zig.Server.Message.EmitBinPath;
+                const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
+                if (!ebp_hdr.flags.cache_hit) {
+                    std.log.info("source changes detected; rebuilding wasm component", .{});
+                }
+                result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+            },
+            else => {}, // ignore other messages
+        }
+
+        stdout.discard(body.len);
+    }
+
+    const stderr = poller.fifo(.stderr);
+    if (stderr.readableLength() > 0) {
+        const owned_stderr = try stderr.toOwnedSlice();
+        defer gpa.free(owned_stderr);
+        std.debug.print("{s}", .{owned_stderr});
+    }
+
+    // Send EOF to stdin.
+    child.stdin.?.close();
+    child.stdin = null;
+
+    switch (try child.wait()) {
+        .Exited => |code| {
+            if (code != 0) {
+                std.log.err(
+                    "the following command exited with error code {d}:\n{s}",
+                    .{ code, try std.Build.Step.allocPrintCmd(arena, null, argv.items) },
+                );
+                return error.AlreadyReported;
+            }
+        },
+        .Signal, .Stopped, .Unknown => {
+            std.log.err(
+                "the following command terminated unexpectedly:\n{s}",
+                .{try std.Build.Step.allocPrintCmd(arena, null, argv.items)},
+            );
+            return error.AlreadyReported;
+        },
+    }
+
+    if (result_error_bundle.errorMessageCount() > 0) {
+        const color = std.zig.Color.auto;
+        result_error_bundle.renderToStdErr(color.renderOptions());
+        std.log.err("the following command failed with {d} compilation errors:\n{s}", .{
+            result_error_bundle.errorMessageCount(),
+            try std.Build.Step.allocPrintCmd(arena, null, argv.items),
+        });
+        return error.AlreadyReported;
+    }
+
+    return result orelse {
+        std.log.err("child process failed to report result\n{s}", .{
+            try std.Build.Step.allocPrintCmd(arena, null, argv.items),
+        });
+        return error.AlreadyReported;
+    };
+}
+
+fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
+    const header: std.zig.Client.Message.Header = .{
+        .tag = tag,
+        .bytes_len = 0,
+    };
+    try file.writeAll(std.mem.asBytes(&header));
 }
 
 fn openBrowserTab(gpa: Allocator, url: []const u8) !void {