update tests, pkghash

build.zig

@@ -29,7 +29,9 @@ pub fn build(b: *std.build.Builder) !void {
     });
 
     facil_lib.linkLibrary(facil_dep.artifact("facil.io"));
-    facil_lib.install();
+    // facil_lib.install();
+    const unused_facil_install_step = b.addInstallArtifact(facil_lib);
+    _ = unused_facil_install_step;
 
     inline for ([_]struct {
         name: []const u8,
@@ -73,10 +75,10 @@ pub fn build(b: *std.build.Builder) !void {
         });
 
         example.linkLibrary(facil_dep.artifact("facil.io"));
-
         example.addModule("zap", zap_module);
 
-        const example_run = example.run();
+        // const example_run = example.run();
+        const example_run = b.addRunArtifact(example);
         example_run_step.dependOn(&example_run.step);
 
         // install the artifact - depending on the "example"
@@ -89,6 +91,7 @@ pub fn build(b: *std.build.Builder) !void {
     //
 
     // authenticating http client for internal testing
+    // (facil.io based, not used anymore)
     //
     var http_client_exe = b.addExecutable(.{
         .name = "http_client",
@@ -128,6 +131,20 @@ pub fn build(b: *std.build.Builder) !void {
     auth_tests.addModule("zap", zap_module);
     auth_tests.step.dependOn(&http_client_runner_build_step.step);
 
+    const run_auth_tests = b.addRunArtifact(auth_tests);
+
     const test_step = b.step("test", "Run unit tests");
-    test_step.dependOn(&auth_tests.step);
+    test_step.dependOn(&run_auth_tests.step);
+
+    // pkghash
+    //
+    var pkghash_exe = b.addExecutable(.{
+        .name = "pkghash",
+        .root_source_file = .{ .path = "./tools/pkghash.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+    var pkghash_step = b.step("pkghash", "Build pkghash");
+    const pkghash_build_step = b.addInstallArtifact(pkghash_exe);
+    pkghash_step.dependOn(&pkghash_build_step.step);
 }

build.zig.zon

@@ -4,8 +4,16 @@
 
     .dependencies = .{
         .@"facil.io" = .{
-            .url = "https://github.com/zigzap/facil.io/archive/64a3fb6d66225d3ff6194e84623eb9d48bc3b6fb.tar.gz",
+            // temp workaround until zig's fetch is fixed, supporting GH's redirects 
-            .hash = "1220da26a9450eb75ecdb93b5dd3dabfea53053734cb68c748f0426d445179bc7c92",
+            .url = "http://localhost:8000/zap-0.0.7.tar.gz",
+
+            // this is how it should be:
+            //.url = "https://github.com/zigzap/facil.io/archive/refs/tags/zap-0.0.7.tar.gz",
+            .hash = "1220d03e0579bbb726efb8224ea289b26227bc421158b45c1b16a60b31bfa400ab33",
+            // our tool says:
+            //.hash = "1220bbc4738c846f3253ae98e1848514f2ed1f02ecc1c7a62076b6508f449e9b5f66",
+
+
         }
     }
 }

flake.lock

@@ -105,11 +105,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1682110011,
+        "lastModified": 1682121798,
-        "narHash": "sha256-J1ArhCRJov3Ycflq7QcmpOzeqqYj39AjlcH77cUx/pQ=",
+        "narHash": "sha256-jfQALrsRCIKFcS9JB5lpsqhifRXHSaslrtTENY+OnCY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "624f65a2b164bc9fe47324606940ffe773196813",
+        "rev": "c3895680d29301a78dd19be91f9c1728c1c28df5",
         "type": "github"
       },
       "original": {
@@ -166,11 +166,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1682078883,
+        "lastModified": 1682122954,
-        "narHash": "sha256-+SEv1PYV/Y84JpwjBw2+SuZQo+YZ/M6n+bMnLTSkAEo=",
+        "narHash": "sha256-CliO/Z30NEQyqghbSWHknU7Fj2MHYLMN9bz88B5JBWo=",
         "owner": "mitchellh",
         "repo": "zig-overlay",
-        "rev": "457d4283ced353eb9be1a1a78a35a37994bf1512",
+        "rev": "803cc45ba8cbd8ce17bc1c8184c2cd3e37be3171",
         "type": "github"
       },
       "original": {

src/test_auth.zig

@@ -122,66 +122,44 @@ fn endpoint_http_unauthorized(e: *Endpoints.SimpleEndpoint, r: zap.SimpleRequest
 //
 // http client code for in-process sending of http request
 //
-fn setHeader(h: [*c]fio.http_s, name: []const u8, value: []const u8) !void {
-    const hname: fio.fio_str_info_s = .{
-        .data = util.toCharPtr(name),
-        .len = name.len,
-        .capa = name.len,
-    };
 
-    const vname: fio.fio_str_info_s = .{
+const ClientAuthReqHeaderFields = struct {
-        .data = util.toCharPtr(value),
+    auth: Authenticators.AuthScheme,
-        .len = value.len,
+    token: []const u8,
-        .capa = value.len,
+};
-    };
-    const ret = fio.http_set_header2(h, hname, vname);
 
-    if (ret == 0) return;
+fn makeRequest(a: std.mem.Allocator, url: []const u8, auth: ?ClientAuthReqHeaderFields) !void {
-    return zap.HttpError.HttpSetHeader;
+    const uri = try std.Uri.parse(url);
-}
 
-fn sendRequest() void {
+    var h = std.http.Headers{ .allocator = a };
-    const ret = zap.http_connect("http://127.0.0.1:3000/test", null, .{
+    defer h.deinit();
-        .on_response = on_response,
-        .on_request = null,
-        .on_upgrade = null,
-        .on_finish = null,
-        .udata = null,
-        .public_folder = null,
-        .public_folder_length = 0,
-        .max_header_size = 32 * 1024,
-        .max_body_size = 500 * 1024,
-        .max_clients = 1,
-        .tls = null,
-        .reserved1 = 0,
-        .reserved2 = 0,
-        .reserved3 = 0,
-        .ws_max_msg_size = 0,
-        .timeout = 5,
-        .ws_timeout = 0,
-        .log = 0,
-        .is_client = 1,
-    });
-    // _ = ret;
-    std.debug.print("\nret = {d}\n", .{ret});
-    zap.fio_start(.{ .threads = 1, .workers = 1 });
-}
 
-fn on_response(r: [*c]fio.http_s) callconv(.C) void {
+    if (auth) |auth_fields| {
-    // the first time around, we need to complete the request. E.g. set headers.
+        const authstring = try std.fmt.allocPrint(a, "{s}{s}", .{ auth_fields.auth.str(), auth_fields.token });
-    if (r.*.status_str == zap.FIOBJ_INVALID) {
+        defer a.free(authstring);
-        setHeader(r, "Authorization", "Bearer ABCDEFG") catch return;
+        try h.append(auth_fields.auth.headerFieldStrHeader(), authstring);
-        zap.http_finish(r);
-        return;
-    }
-    const response = zap.http_req2str(r);
-    if (zap.fio2str(response)) |body| {
-        std.debug.print("{s}\n", .{body});
-    } else {
-        std.debug.print("Oops\n", .{});
     }
+
+    var http_client: std.http.Client = .{ .allocator = a };
+    defer http_client.deinit();
+
+    var req = try http_client.request(.GET, uri, h, .{});
+    defer req.deinit();
+
+    try req.start();
+    try req.do();
+    // var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
+    // var buffer: [1024]u8 = undefined;
+    // we know we won't receive a lot
+    // const len = try br.reader().readAll(&buffer);
+    // std.debug.print("RESPONSE:\n{s}\n", .{buffer[0..len]});
     zap.fio_stop();
 }
+
+fn makeRequestThread(a: std.mem.Allocator, url: []const u8, auth: ?ClientAuthReqHeaderFields) !std.Thread {
+    return try std.Thread.spawn(.{}, makeRequest, .{ a, url, auth });
+}
+
 //
 // end of http client code
 //
@@ -190,32 +168,6 @@ test "BearerAuthSingle authenticateRequest OK" {
     const a = std.testing.allocator;
     const token = "ABCDEFG";
 
-    //
-    // Unfortunately, spawning a child process confuses facilio:
-    //
-    // 1. attempt: spawn curl process before we start facilio threads
-    // this doesn't work: facilio doesn't start up if we spawn a child process
-    // var p = std.ChildProcess.init(&.{
-    //     "bash",
-    //     "-c",
-    //     "sleep 10; curl -H \"Authorization: Bearer\"" ++ token ++ " http://localhost:3000/test -v",
-    // }, a);
-    // try p.spawn();
-
-    // 2. attempt:
-    // our custom client doesn't work either
-    // var p = std.ChildProcess.init(&.{
-    //     "bash",
-    //     "-c",
-    //     "sleep 3; ./zig-out/bin/http_client &",
-    // }, a);
-    // try p.spawn();
-    // std.debug.print("done spawning\n", .{});
-
-    // 3. attempt: sending the request in-process
-    // this doesn't work either because facilio wants to be either server or client, gets confused, at least when we're doing it this way
-    // sendRequest();
-
     // setup listener
     var listener = zap.SimpleEndpointListener.init(
         a,
@@ -248,10 +200,13 @@ test "BearerAuthSingle authenticateRequest OK" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("\n\n*******************************************\n", .{});
+    // std.debug.print("\n\n*******************************************\n", .{});
-    std.debug.print("\n\nPlease run the following:\n", .{});
+    // std.debug.print("\n\nPlease run the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client_runner\n", .{});
+    // std.debug.print("./zig-out/bin/http_client_runner\n", .{});
-    std.debug.print("\n\n*******************************************\n", .{});
+    // std.debug.print("\n\n*******************************************\n", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = token });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -304,8 +259,11 @@ test "BearerAuthSingle authenticateRequest test-unauthorized" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Bearer invalid\r", .{});
+    // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Bearer invalid\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = "invalid" });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -352,8 +310,11 @@ test "BearerAuthMulti authenticateRequest OK" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{});
+    // std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer " ++ token ++ "\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = token });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -400,8 +361,11 @@ test "BearerAuthMulti authenticateRequest test-unauthorized" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{});
+    // std.debug.print("./zig-out/bin/http_client_runner http://127.0.0.1:3000/test Bearer invalid\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Bearer, .token = "invalid" });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -453,8 +417,11 @@ test "BasicAuth Token68 authenticateRequest" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ token ++ "\r", .{});
+    // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ token ++ "\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = token });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -506,8 +473,11 @@ test "BasicAuth Token68 authenticateRequest test-unauthorized" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ "invalid\r", .{});
+    // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic " ++ "invalid\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = "invalid" });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -569,8 +539,11 @@ test "BasicAuth UserPass authenticateRequest" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}\r", .{encoded});
+    // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}\r", .{encoded});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = encoded });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{
@@ -619,6 +592,7 @@ test "BasicAuth UserPass authenticateRequest test-unauthorized" {
     var encoder = std.base64.url_safe.Encoder;
     var buffer: [256]u8 = undefined;
     const encoded = encoder.encode(&buffer, token);
+    _ = encoded;
 
     // create authenticator
     const Authenticator = Authenticators.BasicAuth(Map, .UserPass);
@@ -632,8 +606,11 @@ test "BasicAuth UserPass authenticateRequest test-unauthorized" {
     try listener.addEndpoint(auth_ep.getEndpoint());
 
     listener.listen() catch {};
-    std.debug.print("Waiting for the following:\n", .{});
+    // std.debug.print("Waiting for the following:\n", .{});
-    std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic {s}-invalid\r", .{encoded});
+    // std.debug.print("./zig-out/bin/http_client http://127.0.0.1:3000/test Basic invalid\r", .{});
+
+    const thread = try makeRequestThread(a, "http://127.0.0.1:3000/test", .{ .auth = .Basic, .token = "invalid" });
+    defer thread.join();
 
     // start worker threads
     zap.start(.{

tools/Manifest.zig

@@ -0,0 +1,500 @@
+// borrowed from the zig sourcebase https://github.com/ziglang/zig
+pub const basename = "build.zig.zon";
+pub const Hash = std.crypto.hash.sha2.Sha256;
+
+pub const Dependency = struct {
+    url: []const u8,
+    url_tok: Ast.TokenIndex,
+    hash: ?[]const u8,
+    hash_tok: Ast.TokenIndex,
+};
+
+pub const ErrorMessage = struct {
+    msg: []const u8,
+    tok: Ast.TokenIndex,
+    off: u32,
+};
+
+pub const MultihashFunction = enum(u16) {
+    identity = 0x00,
+    sha1 = 0x11,
+    @"sha2-256" = 0x12,
+    @"sha2-512" = 0x13,
+    @"sha3-512" = 0x14,
+    @"sha3-384" = 0x15,
+    @"sha3-256" = 0x16,
+    @"sha3-224" = 0x17,
+    @"sha2-384" = 0x20,
+    @"sha2-256-trunc254-padded" = 0x1012,
+    @"sha2-224" = 0x1013,
+    @"sha2-512-224" = 0x1014,
+    @"sha2-512-256" = 0x1015,
+    @"blake2b-256" = 0xb220,
+    _,
+};
+
+pub const multihash_function: MultihashFunction = switch (Hash) {
+    std.crypto.hash.sha2.Sha256 => .@"sha2-256",
+    else => @compileError("unreachable"),
+};
+comptime {
+    // We avoid unnecessary uleb128 code in hexDigest by asserting here the
+    // values are small enough to be contained in the one-byte encoding.
+    assert(@enumToInt(multihash_function) < 127);
+    assert(Hash.digest_length < 127);
+}
+pub const multihash_len = 1 + 1 + Hash.digest_length;
+
+name: []const u8,
+version: std.SemanticVersion,
+dependencies: std.StringArrayHashMapUnmanaged(Dependency),
+
+errors: []ErrorMessage,
+arena_state: std.heap.ArenaAllocator.State,
+
+pub const Error = Allocator.Error;
+
+pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest {
+    const node_tags = ast.nodes.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+    assert(node_tags[0] == .root);
+    const main_node_index = node_datas[0].lhs;
+
+    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+    errdefer arena_instance.deinit();
+
+    var p: Parse = .{
+        .gpa = gpa,
+        .ast = ast,
+        .arena = arena_instance.allocator(),
+        .errors = .{},
+
+        .name = undefined,
+        .version = undefined,
+        .dependencies = .{},
+        .buf = .{},
+    };
+    defer p.buf.deinit(gpa);
+    defer p.errors.deinit(gpa);
+    defer p.dependencies.deinit(gpa);
+
+    p.parseRoot(main_node_index) catch |err| switch (err) {
+        error.ParseFailure => assert(p.errors.items.len > 0),
+        else => |e| return e,
+    };
+
+    return .{
+        .name = p.name,
+        .version = p.version,
+        .dependencies = try p.dependencies.clone(p.arena),
+        .errors = try p.arena.dupe(ErrorMessage, p.errors.items),
+        .arena_state = arena_instance.state,
+    };
+}
+
+pub fn deinit(man: *Manifest, gpa: Allocator) void {
+    man.arena_state.promote(gpa).deinit();
+    man.* = undefined;
+}
+
+const hex_charset = "0123456789abcdef";
+
+pub fn hex64(x: u64) [16]u8 {
+    var result: [16]u8 = undefined;
+    var i: usize = 0;
+    while (i < 8) : (i += 1) {
+        const byte = @truncate(u8, x >> @intCast(u6, 8 * i));
+        result[i * 2 + 0] = hex_charset[byte >> 4];
+        result[i * 2 + 1] = hex_charset[byte & 15];
+    }
+    return result;
+}
+
+test hex64 {
+    const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
+    try std.testing.expectEqualStrings("[00efcdab78563412]", s);
+}
+
+pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
+    var result: [multihash_len * 2]u8 = undefined;
+
+    result[0] = hex_charset[@enumToInt(multihash_function) >> 4];
+    result[1] = hex_charset[@enumToInt(multihash_function) & 15];
+
+    result[2] = hex_charset[Hash.digest_length >> 4];
+    result[3] = hex_charset[Hash.digest_length & 15];
+
+    for (digest, 0..) |byte, i| {
+        result[4 + i * 2] = hex_charset[byte >> 4];
+        result[5 + i * 2] = hex_charset[byte & 15];
+    }
+    return result;
+}
+
+const Parse = struct {
+    gpa: Allocator,
+    ast: std.zig.Ast,
+    arena: Allocator,
+    buf: std.ArrayListUnmanaged(u8),
+    errors: std.ArrayListUnmanaged(ErrorMessage),
+
+    name: []const u8,
+    version: std.SemanticVersion,
+    dependencies: std.StringArrayHashMapUnmanaged(Dependency),
+
+    const InnerError = error{ ParseFailure, OutOfMemory };
+
+    fn parseRoot(p: *Parse, node: Ast.Node.Index) !void {
+        const ast = p.ast;
+        const main_tokens = ast.nodes.items(.main_token);
+        const main_token = main_tokens[node];
+
+        var buf: [2]Ast.Node.Index = undefined;
+        const struct_init = ast.fullStructInit(&buf, node) orelse {
+            return fail(p, main_token, "expected top level expression to be a struct", .{});
+        };
+
+        var have_name = false;
+        var have_version = false;
+
+        for (struct_init.ast.fields) |field_init| {
+            const name_token = ast.firstToken(field_init) - 2;
+            const field_name = try identifierTokenString(p, name_token);
+            // We could get fancy with reflection and comptime logic here but doing
+            // things manually provides an opportunity to do any additional verification
+            // that is desirable on a per-field basis.
+            if (mem.eql(u8, field_name, "dependencies")) {
+                try parseDependencies(p, field_init);
+            } else if (mem.eql(u8, field_name, "name")) {
+                p.name = try parseString(p, field_init);
+                have_name = true;
+            } else if (mem.eql(u8, field_name, "version")) {
+                const version_text = try parseString(p, field_init);
+                p.version = std.SemanticVersion.parse(version_text) catch |err| v: {
+                    try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
+                    break :v undefined;
+                };
+                have_version = true;
+            } else {
+                // Ignore unknown fields so that we can add fields in future zig
+                // versions without breaking older zig versions.
+            }
+        }
+
+        if (!have_name) {
+            try appendError(p, main_token, "missing top-level 'name' field", .{});
+        }
+
+        if (!have_version) {
+            try appendError(p, main_token, "missing top-level 'version' field", .{});
+        }
+    }
+
+    fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
+        const ast = p.ast;
+        const main_tokens = ast.nodes.items(.main_token);
+
+        var buf: [2]Ast.Node.Index = undefined;
+        const struct_init = ast.fullStructInit(&buf, node) orelse {
+            const tok = main_tokens[node];
+            return fail(p, tok, "expected dependencies expression to be a struct", .{});
+        };
+
+        for (struct_init.ast.fields) |field_init| {
+            const name_token = ast.firstToken(field_init) - 2;
+            const dep_name = try identifierTokenString(p, name_token);
+            const dep = try parseDependency(p, field_init);
+            try p.dependencies.put(p.gpa, dep_name, dep);
+        }
+    }
+
+    fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
+        const ast = p.ast;
+        const main_tokens = ast.nodes.items(.main_token);
+
+        var buf: [2]Ast.Node.Index = undefined;
+        const struct_init = ast.fullStructInit(&buf, node) orelse {
+            const tok = main_tokens[node];
+            return fail(p, tok, "expected dependency expression to be a struct", .{});
+        };
+
+        var dep: Dependency = .{
+            .url = undefined,
+            .url_tok = undefined,
+            .hash = null,
+            .hash_tok = undefined,
+        };
+        var have_url = false;
+
+        for (struct_init.ast.fields) |field_init| {
+            const name_token = ast.firstToken(field_init) - 2;
+            const field_name = try identifierTokenString(p, name_token);
+            // We could get fancy with reflection and comptime logic here but doing
+            // things manually provides an opportunity to do any additional verification
+            // that is desirable on a per-field basis.
+            if (mem.eql(u8, field_name, "url")) {
+                dep.url = parseString(p, field_init) catch |err| switch (err) {
+                    error.ParseFailure => continue,
+                    else => |e| return e,
+                };
+                dep.url_tok = main_tokens[field_init];
+                have_url = true;
+            } else if (mem.eql(u8, field_name, "hash")) {
+                dep.hash = parseHash(p, field_init) catch |err| switch (err) {
+                    error.ParseFailure => continue,
+                    else => |e| return e,
+                };
+                dep.hash_tok = main_tokens[field_init];
+            } else {
+                // Ignore unknown fields so that we can add fields in future zig
+                // versions without breaking older zig versions.
+            }
+        }
+
+        if (!have_url) {
+            try appendError(p, main_tokens[node], "dependency is missing 'url' field", .{});
+        }
+
+        return dep;
+    }
+
+    fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
+        const ast = p.ast;
+        const node_tags = ast.nodes.items(.tag);
+        const main_tokens = ast.nodes.items(.main_token);
+        if (node_tags[node] != .string_literal) {
+            return fail(p, main_tokens[node], "expected string literal", .{});
+        }
+        const str_lit_token = main_tokens[node];
+        const token_bytes = ast.tokenSlice(str_lit_token);
+        p.buf.clearRetainingCapacity();
+        try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
+        const duped = try p.arena.dupe(u8, p.buf.items);
+        return duped;
+    }
+
+    fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
+        const ast = p.ast;
+        const main_tokens = ast.nodes.items(.main_token);
+        const tok = main_tokens[node];
+        const h = try parseString(p, node);
+
+        if (h.len >= 2) {
+            const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
+                return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
+                    @errorName(err),
+                });
+            };
+            if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) {
+                return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
+            }
+        }
+
+        const hex_multihash_len = 2 * Manifest.multihash_len;
+        if (h.len != hex_multihash_len) {
+            return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
+                hex_multihash_len, h.len,
+            });
+        }
+
+        return h;
+    }
+
+    /// TODO: try to DRY this with AstGen.identifierTokenString
+    fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
+        const ast = p.ast;
+        const token_tags = ast.tokens.items(.tag);
+        assert(token_tags[token] == .identifier);
+        const ident_name = ast.tokenSlice(token);
+        if (!mem.startsWith(u8, ident_name, "@")) {
+            return ident_name;
+        }
+        p.buf.clearRetainingCapacity();
+        try parseStrLit(p, token, &p.buf, ident_name, 1);
+        const duped = try p.arena.dupe(u8, p.buf.items);
+        return duped;
+    }
+
+    /// TODO: try to DRY this with AstGen.parseStrLit
+    fn parseStrLit(
+        p: *Parse,
+        token: Ast.TokenIndex,
+        buf: *std.ArrayListUnmanaged(u8),
+        bytes: []const u8,
+        offset: u32,
+    ) InnerError!void {
+        const raw_string = bytes[offset..];
+        var buf_managed = buf.toManaged(p.gpa);
+        const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
+        buf.* = buf_managed.moveToUnmanaged();
+        switch (try result) {
+            .success => {},
+            .failure => |err| try p.appendStrLitError(err, token, bytes, offset),
+        }
+    }
+
+    /// TODO: try to DRY this with AstGen.failWithStrLitError
+    fn appendStrLitError(
+        p: *Parse,
+        err: std.zig.string_literal.Error,
+        token: Ast.TokenIndex,
+        bytes: []const u8,
+        offset: u32,
+    ) Allocator.Error!void {
+        const raw_string = bytes[offset..];
+        switch (err) {
+            .invalid_escape_character => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "invalid escape character: '{c}'",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .expected_hex_digit => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "expected hex digit, found '{c}'",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .empty_unicode_escape_sequence => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "empty unicode escape sequence",
+                    .{},
+                );
+            },
+            .expected_hex_digit_or_rbrace => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "expected hex digit or '}}', found '{c}'",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .invalid_unicode_codepoint => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "unicode escape does not correspond to a valid codepoint",
+                    .{},
+                );
+            },
+            .expected_lbrace => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "expected '{{', found '{c}",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .expected_rbrace => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "expected '}}', found '{c}",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .expected_single_quote => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "expected single quote ('), found '{c}",
+                    .{raw_string[bad_index]},
+                );
+            },
+            .invalid_character => |bad_index| {
+                try p.appendErrorOff(
+                    token,
+                    offset + @intCast(u32, bad_index),
+                    "invalid byte in string or character literal: '{c}'",
+                    .{raw_string[bad_index]},
+                );
+            },
+        }
+    }
+
+    fn fail(
+        p: *Parse,
+        tok: Ast.TokenIndex,
+        comptime fmt: []const u8,
+        args: anytype,
+    ) InnerError {
+        try appendError(p, tok, fmt, args);
+        return error.ParseFailure;
+    }
+
+    fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
+        return appendErrorOff(p, tok, 0, fmt, args);
+    }
+
+    fn appendErrorOff(
+        p: *Parse,
+        tok: Ast.TokenIndex,
+        byte_offset: u32,
+        comptime fmt: []const u8,
+        args: anytype,
+    ) Allocator.Error!void {
+        try p.errors.append(p.gpa, .{
+            .msg = try std.fmt.allocPrint(p.arena, fmt, args),
+            .tok = tok,
+            .off = byte_offset,
+        });
+    }
+};
+
+const Manifest = @This();
+const std = @import("std");
+const mem = std.mem;
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const Ast = std.zig.Ast;
+const testing = std.testing;
+
+test "basic" {
+    const gpa = testing.allocator;
+
+    const example =
+        \\.{
+        \\    .name = "foo",
+        \\    .version = "3.2.1",
+        \\    .dependencies = .{
+        \\        .bar = .{
+        \\            .url = "https://example.com/baz.tar.gz",
+        \\            .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
+        \\        },
+        \\    },
+        \\}
+    ;
+
+    var ast = try std.zig.Ast.parse(gpa, example, .zon);
+    defer ast.deinit(gpa);
+
+    try testing.expect(ast.errors.len == 0);
+
+    var manifest = try Manifest.parse(gpa, ast);
+    defer manifest.deinit(gpa);
+
+    try testing.expectEqualStrings("foo", manifest.name);
+
+    try testing.expectEqual(@as(std.SemanticVersion, .{
+        .major = 3,
+        .minor = 2,
+        .patch = 1,
+    }), manifest.version);
+
+    try testing.expect(manifest.dependencies.count() == 1);
+    try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
+    try testing.expectEqualStrings(
+        "https://example.com/baz.tar.gz",
+        manifest.dependencies.values()[0].url,
+    );
+    try testing.expectEqualStrings(
+        "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
+        manifest.dependencies.values()[0].hash orelse return error.TestFailed,
+    );
+}

tools/pkghash.zig

@@ -0,0 +1,233 @@
+const std = @import("std");
+const builtin = std.builtin;
+const assert = std.debug.assert;
+const io = std.io;
+const fs = std.fs;
+const mem = std.mem;
+const process = std.process;
+const Allocator = mem.Allocator;
+const ThreadPool = std.Thread.Pool;
+const WaitGroup = std.Thread.WaitGroup;
+
+const Manifest = @import("Manifest.zig");
+
+var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
+
+pub fn fatal(comptime format: []const u8, args: anytype) noreturn {
+    std.log.err(format, args);
+    process.exit(1);
+}
+
+pub fn main() !void {
+    const gpa = general_purpose_allocator.allocator();
+    defer _ = general_purpose_allocator.deinit();
+    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+    defer arena_instance.deinit();
+    const arena = arena_instance.allocator();
+
+    const args = try process.argsAlloc(arena);
+    try cmdPkg(gpa, arena, args);
+}
+pub const usage_pkg =
+    \\Usage: zig pkg [command] [options]
+    \\
+    \\Options: 
+    \\  -h --help           Print this help and exit.
+    \\
+    \\Sub-options for [hash]:
+    \\  --allow-directory : calc hash even if no build.zig is present
+    \\  
+;
+
+pub fn cmdPkg(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
+    _ = arena;
+    if (args.len == 0) fatal("Expected at least one argument.\n", .{});
+    const command_arg = args[0];
+
+    if (mem.eql(u8, command_arg, "-h") or mem.eql(u8, command_arg, "--help")) {
+        const stdout = io.getStdOut().writer();
+        try stdout.writeAll(usage_pkg);
+        return;
+    }
+
+    const cwd = std.fs.cwd();
+
+    dir_test: {
+        if (args.len > 1 and mem.eql(u8, args[1], "--allow-directory")) break :dir_test;
+        try if (cwd.access("build.zig", .{})) |_| break :dir_test else |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| e,
+        };
+        try if (cwd.access("build.zig.zon", .{})) |_| break :dir_test else |err| switch (err) {
+            error.FileNotFound => {},
+            else => |e| e,
+        };
+        break :dir_test fatal("Could not find either build.zig or build.zig.zon in this directory.\n Use --allow-directory to override this check.\n", .{});
+    }
+
+    const hash = blk: {
+        const cwd_absolute_path = try cwd.realpathAlloc(gpa, ".");
+        defer gpa.free(cwd_absolute_path);
+
+        // computePackageHash will close the directory after completion
+        // std.debug.print("abspath: {s}\n", .{cwd_absolute_path});
+        var cwd_copy = try fs.openIterableDirAbsolute(cwd_absolute_path, .{});
+        errdefer cwd_copy.dir.close();
+
+        var thread_pool: ThreadPool = undefined;
+        try thread_pool.init(.{ .allocator = gpa });
+        defer thread_pool.deinit();
+
+        // workaround for missing inclusion/exclusion support -> #14311.
+        const excluded_directories: []const []const u8 = &.{
+            "zig-out",
+            "zig-cache",
+            ".git",
+        };
+        break :blk try computePackageHashExcludingDirectories(
+            &thread_pool,
+            .{ .dir = cwd_copy.dir },
+            excluded_directories,
+        );
+    };
+
+    const std_out = std.io.getStdOut();
+    const digest = Manifest.hexDigest(hash);
+    try std_out.writeAll(digest[0..]);
+    try std_out.writeAll("\n");
+}
+
+/// Make a file system path identical independently of operating system path inconsistencies.
+/// This converts backslashes into forward slashes.
+fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
+    const canonical_sep = '/';
+
+    if (fs.path.sep == canonical_sep)
+        return fs_path;
+
+    const normalized = try arena.dupe(u8, fs_path);
+    for (normalized) |*byte| {
+        switch (byte.*) {
+            fs.path.sep => byte.* = canonical_sep,
+            else => continue,
+        }
+    }
+    return normalized;
+}
+
+const HashedFile = struct {
+    fs_path: []const u8,
+    normalized_path: []const u8,
+    hash: [Manifest.Hash.digest_length]u8,
+    failure: Error!void,
+
+    const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
+
+    fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
+        _ = context;
+        return mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
+    }
+};
+
+fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
+    defer wg.finish();
+    hashed_file.failure = hashFileFallible(dir, hashed_file);
+}
+
+fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
+    var buf: [8000]u8 = undefined;
+    var file = try dir.openFile(hashed_file.fs_path, .{});
+    defer file.close();
+    var hasher = Manifest.Hash.init(.{});
+    hasher.update(hashed_file.normalized_path);
+    hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) });
+    while (true) {
+        const bytes_read = try file.read(&buf);
+        if (bytes_read == 0) break;
+        hasher.update(buf[0..bytes_read]);
+    }
+    hasher.final(&hashed_file.hash);
+}
+
+fn isExecutable(file: fs.File) !bool {
+    _ = file;
+    // hack: in order to mimic current zig's tar extraction, we set everything to
+    // NOT EXECUTABLE
+    // const stat = try file.stat();
+    // return (stat.mode & std.os.S.IXUSR) != 0;
+    return false;
+}
+
+pub fn computePackageHashExcludingDirectories(
+    thread_pool: *ThreadPool,
+    pkg_dir: fs.IterableDir,
+    excluded_directories: []const []const u8,
+) ![Manifest.Hash.digest_length]u8 {
+    const gpa = thread_pool.allocator;
+
+    // We'll use an arena allocator for the path name strings since they all
+    // need to be in memory for sorting.
+    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+    defer arena_instance.deinit();
+    const arena = arena_instance.allocator();
+
+    // Collect all files, recursively, then sort.
+    var all_files = std.ArrayList(*HashedFile).init(gpa);
+    defer all_files.deinit();
+
+    var walker = try pkg_dir.walk(gpa);
+    defer walker.deinit();
+
+    {
+        // The final hash will be a hash of each file hashed independently. This
+        // allows hashing in parallel.
+        var wait_group: WaitGroup = .{};
+        defer wait_group.wait();
+
+        loop: while (try walker.next()) |entry| {
+            switch (entry.kind) {
+                .Directory => {
+                    for (excluded_directories) |dir_name| {
+                        if (mem.eql(u8, entry.basename, dir_name)) {
+                            var item = walker.stack.pop();
+                            if (walker.stack.items.len != 0) {
+                                item.iter.dir.close();
+                            }
+                            continue :loop;
+                        }
+                    }
+                    continue :loop;
+                },
+                .File => {},
+                else => return error.IllegalFileTypeInPackage,
+            }
+            const hashed_file = try arena.create(HashedFile);
+            const fs_path = try arena.dupe(u8, entry.path);
+            hashed_file.* = .{
+                .fs_path = fs_path,
+                .normalized_path = try normalizePath(arena, fs_path),
+                .hash = undefined, // to be populated by the worker
+                .failure = undefined, // to be populated by the worker
+            };
+            wait_group.start();
+            try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
+
+            try all_files.append(hashed_file);
+        }
+    }
+
+    std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan);
+
+    var hasher = Manifest.Hash.init(.{});
+    var any_failures = false;
+    for (all_files.items) |hashed_file| {
+        hashed_file.failure catch |err| {
+            any_failures = true;
+            std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
+        };
+        // std.debug.print("{s} : {s}\n", .{ hashed_file.normalized_path, Manifest.hexDigest(hashed_file.hash) });
+        hasher.update(&hashed_file.hash);
+    }
+    if (any_failures) return error.PackageHashUnavailable;
+    return hasher.finalResult();
+}

tools/server.py

@@ -0,0 +1,19 @@
+import http.server as SimpleHTTPServer
+import socketserver as SocketServer
+import logging
+
+PORT = 8080
+
+class GetHandler(
+        SimpleHTTPServer.SimpleHTTPRequestHandler
+        ):
+
+    def do_GET(self):
+        logging.error(self.headers)
+        SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
+
+
+Handler = GetHandler
+httpd = SocketServer.TCPServer(("", PORT), Handler)
+
+httpd.serve_forever()