From 32cb9462ffa0a9df7a080d67eaf3a5762173f742 Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Mon, 19 Jun 2023 11:21:37 -0400 Subject: [PATCH] std: Support user-provided jsonParse method. Unify json.Parser and json.parse* (#15705) --- lib/std/json.zig | 73 ++++++- lib/std/json/dynamic.zig | 365 ++++++++++---------------------- lib/std/json/dynamic_test.zig | 97 +++------ lib/std/json/static.zig | 353 ++++++++++-------------------- lib/std/json/static_test.zig | 211 +++++++++--------- lib/std/json/test.zig | 18 +- tools/gen_spirv_spec.zig | 13 +- tools/spirv/grammar.zig | 17 ++ tools/update_clang_options.zig | 6 +- tools/update_cpu_features.zig | 6 +- tools/update_spirv_features.zig | 8 +- 11 files changed, 471 insertions(+), 696 deletions(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index 10449cdace..2a12023c76 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1,19 +1,73 @@ //! JSON parsing and stringification conforming to RFC 8259. https://datatracker.ietf.org/doc/html/rfc8259 //! -//! The low-level `Scanner` API reads from an input slice or successive slices of inputs, +//! The low-level `Scanner` API produces `Token`s from an input slice or successive slices of inputs, //! The `Reader` API connects a `std.io.Reader` to a `Scanner`. //! -//! The high-level `parseFromSlice` and `parseFromTokenSource` deserializes a JSON document into a Zig type. -//! The high-level `Parser` parses any JSON document into a dynamically typed `ValueTree` that has its own memory arena. +//! The high-level `parseFromSlice` and `parseFromTokenSource` deserialize a JSON document into a Zig type. +//! Parse into a dynamically-typed `Value` to load any JSON value for runtime inspection. //! //! The low-level `writeStream` emits syntax-conformant JSON tokens to a `std.io.Writer`. -//! The high-level `stringify` serializes a Zig type into JSON. +//! The high-level `stringify` serializes a Zig or `Value` type into JSON. + +const testing = @import("std").testing; +const ArrayList = @import("std").ArrayList; + +test Scanner { + var scanner = Scanner.initCompleteInput(testing.allocator, "{\"foo\": 123}\n"); + defer scanner.deinit(); + try testing.expectEqual(Token.object_begin, try scanner.next()); + try testing.expectEqualSlices(u8, "foo", (try scanner.next()).string); + try testing.expectEqualSlices(u8, "123", (try scanner.next()).number); + try testing.expectEqual(Token.object_end, try scanner.next()); + try testing.expectEqual(Token.end_of_document, try scanner.next()); +} + +test parseFromSlice { + var parsed_str = try parseFromSlice([]const u8, testing.allocator, "\"a\\u0020b\"", .{}); + defer parsed_str.deinit(); + try testing.expectEqualSlices(u8, "a b", parsed_str.value); + + const T = struct { a: i32 = -1, b: [2]u8 }; + var parsed_struct = try parseFromSlice(T, testing.allocator, "{\"b\":\"xy\"}", .{}); + defer parsed_struct.deinit(); + try testing.expectEqual(@as(i32, -1), parsed_struct.value.a); // default value + try testing.expectEqualSlices(u8, "xy", parsed_struct.value.b[0..]); +} + +test Value { + var parsed = try parseFromSlice(Value, testing.allocator, "{\"anything\": \"goes\"}", .{}); + defer parsed.deinit(); + try testing.expectEqualSlices(u8, "goes", parsed.value.object.get("anything").?.string); +} + +test writeStream { + var out = ArrayList(u8).init(testing.allocator); + defer out.deinit(); + var write_stream = writeStream(out.writer(), 99); + try write_stream.beginObject(); + try write_stream.objectField("foo"); + try write_stream.emitNumber(123); + try write_stream.endObject(); + const expected = + \\{ + \\ "foo": 123 + \\} + ; + try testing.expectEqualSlices(u8, expected, out.items); +} + +test stringify { + var out = ArrayList(u8).init(testing.allocator); + defer out.deinit(); + + const T = struct { a: i32, b: []const u8 }; + try stringify(T{ .a = 123, .b = "xy" }, .{}, out.writer()); + try testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.items); +} -pub const ValueTree = @import("json/dynamic.zig").ValueTree; pub const ObjectMap = @import("json/dynamic.zig").ObjectMap; pub const Array = @import("json/dynamic.zig").Array; pub const Value = @import("json/dynamic.zig").Value; -pub const Parser = @import("json/dynamic.zig").Parser; pub const validate = @import("json/scanner.zig").validate; pub const Error = @import("json/scanner.zig").Error; @@ -30,9 +84,11 @@ pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberF pub const ParseOptions = @import("json/static.zig").ParseOptions; pub const parseFromSlice = @import("json/static.zig").parseFromSlice; +pub const parseFromSliceLeaky = @import("json/static.zig").parseFromSliceLeaky; pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource; +pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenSourceLeaky; pub const ParseError = @import("json/static.zig").ParseError; -pub const parseFree = @import("json/static.zig").parseFree; +pub const Parsed = @import("json/static.zig").Parsed; pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions; pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString; @@ -45,6 +101,9 @@ pub const writeStream = @import("json/write_stream.zig").writeStream; // Deprecations pub const parse = @compileError("Deprecated; use parseFromSlice() or parseFromTokenSource() instead."); +pub const parseFree = @compileError("Deprecated; call Parsed(T).deinit() instead."); +pub const Parser = @compileError("Deprecated; use parseFromSlice(Value) or parseFromTokenSource(Value) instead."); +pub const ValueTree = @compileError("Deprecated; use Parsed(Value) instead."); pub const StreamingParser = @compileError("Deprecated; use json.Scanner or json.Reader instead."); pub const TokenStream = @compileError("Deprecated; use json.Scanner or json.Reader instead."); diff --git a/lib/std/json/dynamic.zig b/lib/std/json/dynamic.zig index 057fb93ded..c9bb244181 100644 --- a/lib/std/json/dynamic.zig +++ b/lib/std/json/dynamic.zig @@ -8,26 +8,20 @@ const Allocator = std.mem.Allocator; const StringifyOptions = @import("./stringify.zig").StringifyOptions; const stringify = @import("./stringify.zig").stringify; +const ParseOptions = @import("./static.zig").ParseOptions; +const ParseError = @import("./static.zig").ParseError; + const JsonScanner = @import("./scanner.zig").Scanner; const AllocWhen = @import("./scanner.zig").AllocWhen; const Token = @import("./scanner.zig").Token; const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; -pub const ValueTree = struct { - arena: *ArenaAllocator, - root: Value, - - pub fn deinit(self: *ValueTree) void { - self.arena.deinit(); - self.arena.child_allocator.destroy(self.arena); - } -}; - pub const ObjectMap = StringArrayHashMap(Value); pub const Array = ArrayList(Value); -/// Represents a JSON value -/// Currently only supports numbers that fit into i64 or f64. +/// Represents any JSON value, potentially containing other JSON values. +/// A .float value may be an approximation of the original value. +/// Arbitrary precision numbers can be represented by .number_string values. pub const Value = union(enum) { null, bool: bool, @@ -38,6 +32,33 @@ pub const Value = union(enum) { array: Array, object: ObjectMap, + pub fn parseFromNumberSlice(s: []const u8) Value { + if (!isNumberFormattedLikeAnInteger(s)) { + const f = std.fmt.parseFloat(f64, s) catch unreachable; + if (std.math.isFinite(f)) { + return Value{ .float = f }; + } else { + return Value{ .number_string = s }; + } + } + if (std.fmt.parseInt(i64, s, 10)) |i| { + return Value{ .integer = i }; + } else |e| { + switch (e) { + error.Overflow => return Value{ .number_string = s }, + error.InvalidCharacter => unreachable, + } + } + } + + pub fn dump(self: Value) void { + std.debug.getStderrMutex().lock(); + defer std.debug.getStderrMutex().unlock(); + + const stderr = std.io.getStdErr().writer(); + stringify(self, .{}, stderr) catch return; + } + pub fn jsonStringify( value: @This(), options: StringifyOptions, @@ -80,265 +101,99 @@ pub const Value = union(enum) { } } - pub fn dump(self: Value) void { - std.debug.getStderrMutex().lock(); - defer std.debug.getStderrMutex().unlock(); - - const stderr = std.io.getStdErr().writer(); - stringify(self, .{}, stderr) catch return; - } -}; - -/// A non-stream JSON parser which constructs a tree of Value's. -pub const Parser = struct { - allocator: Allocator, - state: State, - alloc_when: AllocWhen, - // Stores parent nodes and un-combined Values. - stack: Array, - - const State = enum { - object_key, - object_value, - array_value, - simple, - }; - - pub fn init(allocator: Allocator, alloc_when: AllocWhen) Parser { - return Parser{ - .allocator = allocator, - .state = .simple, - .alloc_when = alloc_when, - .stack = Array.init(allocator), - }; - } - - pub fn deinit(p: *Parser) void { - p.stack.deinit(); - } - - pub fn reset(p: *Parser) void { - p.state = .simple; - p.stack.shrinkRetainingCapacity(0); - } - - pub fn parse(p: *Parser, input: []const u8) !ValueTree { - var scanner = JsonScanner.initCompleteInput(p.allocator, input); - defer scanner.deinit(); - - var arena = try p.allocator.create(ArenaAllocator); - errdefer p.allocator.destroy(arena); - - arena.* = ArenaAllocator.init(p.allocator); - errdefer arena.deinit(); - - const allocator = arena.allocator(); + pub fn jsonParse(allocator: Allocator, source: anytype, options: ParseOptions) ParseError(@TypeOf(source.*))!@This() { + _ = options; + // The grammar of the stack is: + // (.array | .object .string)* + var stack = Array.init(allocator); + defer stack.deinit(); while (true) { - const token = try scanner.nextAlloc(allocator, p.alloc_when); - if (token == .end_of_document) break; - try p.transition(allocator, token); - } + // Assert the stack grammar at the top of the stack. + debug.assert(stack.items.len == 0 or + stack.items[stack.items.len - 1] == .array or + (stack.items[stack.items.len - 2] == .object and stack.items[stack.items.len - 1] == .string)); - debug.assert(p.stack.items.len == 1); - - return ValueTree{ - .arena = arena, - .root = p.stack.items[0], - }; - } - - // Even though p.allocator exists, we take an explicit allocator so that allocation state - // can be cleaned up on error correctly during a `parse` on call. - fn transition(p: *Parser, allocator: Allocator, token: Token) !void { - switch (p.state) { - .object_key => switch (token) { - .object_end => { - if (p.stack.items.len == 1) { - return; - } - - var value = p.stack.pop(); - try p.pushToParent(&value); + switch (try source.nextAlloc(allocator, .alloc_if_needed)) { + inline .string, .allocated_string => |s| { + return try handleCompleteValue(&stack, allocator, source, Value{ .string = s }) orelse continue; }, - .string => |s| { - try p.stack.append(Value{ .string = s }); - p.state = .object_value; + inline .number, .allocated_number => |slice| { + return try handleCompleteValue(&stack, allocator, source, Value.parseFromNumberSlice(slice)) orelse continue; }, - .allocated_string => |s| { - try p.stack.append(Value{ .string = s }); - p.state = .object_value; - }, - else => unreachable, - }, - .object_value => { - var object = &p.stack.items[p.stack.items.len - 2].object; - var key = p.stack.items[p.stack.items.len - 1].string; - switch (token) { - .object_begin => { - try p.stack.append(Value{ .object = ObjectMap.init(allocator) }); - p.state = .object_key; - }, - .array_begin => { - try p.stack.append(Value{ .array = Array.init(allocator) }); - p.state = .array_value; - }, - .string => |s| { - try object.put(key, Value{ .string = s }); - _ = p.stack.pop(); - p.state = .object_key; - }, - .allocated_string => |s| { - try object.put(key, Value{ .string = s }); - _ = p.stack.pop(); - p.state = .object_key; - }, - .number => |slice| { - try object.put(key, try p.parseNumber(slice)); - _ = p.stack.pop(); - p.state = .object_key; - }, - .allocated_number => |slice| { - try object.put(key, try p.parseNumber(slice)); - _ = p.stack.pop(); - p.state = .object_key; - }, - .true => { - try object.put(key, Value{ .bool = true }); - _ = p.stack.pop(); - p.state = .object_key; - }, - .false => { - try object.put(key, Value{ .bool = false }); - _ = p.stack.pop(); - p.state = .object_key; - }, - .null => { - try object.put(key, .null); - _ = p.stack.pop(); - p.state = .object_key; - }, - .object_end, .array_end, .end_of_document => unreachable, - .partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable, - } - }, - .array_value => { - var array = &p.stack.items[p.stack.items.len - 1].array; + .null => return try handleCompleteValue(&stack, allocator, source, .null) orelse continue, + .true => return try handleCompleteValue(&stack, allocator, source, Value{ .bool = true }) orelse continue, + .false => return try handleCompleteValue(&stack, allocator, source, Value{ .bool = false }) orelse continue, - switch (token) { - .array_end => { - if (p.stack.items.len == 1) { - return; - } - - var value = p.stack.pop(); - try p.pushToParent(&value); - }, - .object_begin => { - try p.stack.append(Value{ .object = ObjectMap.init(allocator) }); - p.state = .object_key; - }, - .array_begin => { - try p.stack.append(Value{ .array = Array.init(allocator) }); - p.state = .array_value; - }, - .string => |s| { - try array.append(Value{ .string = s }); - }, - .allocated_string => |s| { - try array.append(Value{ .string = s }); - }, - .number => |slice| { - try array.append(try p.parseNumber(slice)); - }, - .allocated_number => |slice| { - try array.append(try p.parseNumber(slice)); - }, - .true => { - try array.append(Value{ .bool = true }); - }, - .false => { - try array.append(Value{ .bool = false }); - }, - .null => { - try array.append(.null); - }, - .object_end, .end_of_document => unreachable, - .partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable, - } - }, - .simple => switch (token) { .object_begin => { - try p.stack.append(Value{ .object = ObjectMap.init(allocator) }); - p.state = .object_key; + switch (try source.nextAlloc(allocator, .alloc_if_needed)) { + .object_end => return try handleCompleteValue(&stack, allocator, source, Value{ .object = ObjectMap.init(allocator) }) orelse continue, + inline .string, .allocated_string => |key| { + try stack.appendSlice(&[_]Value{ + Value{ .object = ObjectMap.init(allocator) }, + Value{ .string = key }, + }); + }, + else => unreachable, + } }, .array_begin => { - try p.stack.append(Value{ .array = Array.init(allocator) }); - p.state = .array_value; + try stack.append(Value{ .array = Array.init(allocator) }); }, - .string => |s| { - try p.stack.append(Value{ .string = s }); - }, - .allocated_string => |s| { - try p.stack.append(Value{ .string = s }); - }, - .number => |slice| { - try p.stack.append(try p.parseNumber(slice)); - }, - .allocated_number => |slice| { - try p.stack.append(try p.parseNumber(slice)); - }, - .true => { - try p.stack.append(Value{ .bool = true }); - }, - .false => { - try p.stack.append(Value{ .bool = false }); - }, - .null => { - try p.stack.append(.null); - }, - .object_end, .array_end, .end_of_document => unreachable, - .partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable, - }, - } - } + .array_end => return try handleCompleteValue(&stack, allocator, source, stack.pop()) orelse continue, - fn pushToParent(p: *Parser, value: *const Value) !void { - switch (p.stack.items[p.stack.items.len - 1]) { - // Object Parent -> [ ..., object, , value ] - .string => |key| { - _ = p.stack.pop(); - - var object = &p.stack.items[p.stack.items.len - 1].object; - try object.put(key, value.*); - p.state = .object_key; - }, - // Array Parent -> [ ..., , value ] - .array => |*array| { - try array.append(value.*); - p.state = .array_value; - }, - else => { - unreachable; - }, - } - } - - fn parseNumber(p: *Parser, slice: []const u8) !Value { - _ = p; - return if (isNumberFormattedLikeAnInteger(slice)) - Value{ - .integer = std.fmt.parseInt(i64, slice, 10) catch |e| switch (e) { - error.Overflow => return Value{ .number_string = slice }, - error.InvalidCharacter => |err| return err, - }, + else => unreachable, } - else - Value{ .float = try std.fmt.parseFloat(f64, slice) }; + } } }; +fn handleCompleteValue(stack: *Array, allocator: Allocator, source: anytype, value_: Value) !?Value { + if (stack.items.len == 0) return value_; + var value = value_; + while (true) { + // Assert the stack grammar at the top of the stack. + debug.assert(stack.items[stack.items.len - 1] == .array or + (stack.items[stack.items.len - 2] == .object and stack.items[stack.items.len - 1] == .string)); + switch (stack.items[stack.items.len - 1]) { + .string => |key| { + // stack: [..., .object, .string] + _ = stack.pop(); + + // stack: [..., .object] + var object = &stack.items[stack.items.len - 1].object; + try object.put(key, value); + + // This is an invalid state to leave the stack in, + // so we have to process the next token before we return. + switch (try source.nextAlloc(allocator, .alloc_if_needed)) { + .object_end => { + // This object is complete. + value = stack.pop(); + // Effectively recurse now that we have a complete value. + if (stack.items.len == 0) return value; + continue; + }, + inline .string, .allocated_string => |next_key| { + // We've got another key. + try stack.append(Value{ .string = next_key }); + // stack: [..., .object, .string] + return null; + }, + else => unreachable, + } + }, + .array => |*array| { + // stack: [..., .array] + try array.append(value); + return null; + }, + else => unreachable, + } + } +} + test { _ = @import("dynamic_test.zig"); } diff --git a/lib/std/json/dynamic_test.zig b/lib/std/json/dynamic_test.zig index f20098f2d7..d764e387af 100644 --- a/lib/std/json/dynamic_test.zig +++ b/lib/std/json/dynamic_test.zig @@ -5,12 +5,14 @@ const testing = std.testing; const ObjectMap = @import("dynamic.zig").ObjectMap; const Array = @import("dynamic.zig").Array; const Value = @import("dynamic.zig").Value; -const Parser = @import("dynamic.zig").Parser; + +const parseFromSlice = @import("static.zig").parseFromSlice; +const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky; +const parseFromTokenSource = @import("static.zig").parseFromTokenSource; + +const jsonReader = @import("scanner.zig").reader; test "json.parser.dynamic" { - var p = Parser.init(testing.allocator, .alloc_if_needed); - defer p.deinit(); - const s = \\{ \\ "Image": { @@ -31,10 +33,10 @@ test "json.parser.dynamic" { \\} ; - var tree = try p.parse(s); - defer tree.deinit(); + var parsed = try parseFromSlice(Value, testing.allocator, s, .{}); + defer parsed.deinit(); - var root = tree.root; + var root = parsed.value; var image = root.object.get("Image").?; @@ -98,22 +100,22 @@ test "write json then parse it" { try jw.endObject(); - var parser = Parser.init(testing.allocator, .alloc_if_needed); - defer parser.deinit(); - var tree = try parser.parse(fixed_buffer_stream.getWritten()); - defer tree.deinit(); + fixed_buffer_stream = std.io.fixedBufferStream(fixed_buffer_stream.getWritten()); + var json_reader = jsonReader(testing.allocator, fixed_buffer_stream.reader()); + defer json_reader.deinit(); + var parsed = try parseFromTokenSource(Value, testing.allocator, &json_reader, .{}); + defer parsed.deinit(); - try testing.expect(tree.root.object.get("f").?.bool == false); - try testing.expect(tree.root.object.get("t").?.bool == true); - try testing.expect(tree.root.object.get("int").?.integer == 1234); - try testing.expect(tree.root.object.get("array").?.array.items[0].null == {}); - try testing.expect(tree.root.object.get("array").?.array.items[1].float == 12.34); - try testing.expect(mem.eql(u8, tree.root.object.get("str").?.string, "hello")); + try testing.expect(parsed.value.object.get("f").?.bool == false); + try testing.expect(parsed.value.object.get("t").?.bool == true); + try testing.expect(parsed.value.object.get("int").?.integer == 1234); + try testing.expect(parsed.value.object.get("array").?.array.items[0].null == {}); + try testing.expect(parsed.value.object.get("array").?.array.items[1].float == 12.34); + try testing.expect(mem.eql(u8, parsed.value.object.get("str").?.string, "hello")); } -fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value { - var p = Parser.init(arena_allocator, .alloc_if_needed); - return (try p.parse(json_str)).root; +fn testParse(allocator: std.mem.Allocator, json_str: []const u8) !Value { + return parseFromSliceLeaky(Value, allocator, json_str, .{}); } test "parsing empty string gives appropriate error" { @@ -122,22 +124,16 @@ test "parsing empty string gives appropriate error" { try testing.expectError(error.UnexpectedEndOfInput, testParse(arena_allocator.allocator(), "")); } -test "parse tree should not contain dangling pointers" { - var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_allocator.deinit(); - - var p = Parser.init(arena_allocator.allocator(), .alloc_if_needed); - defer p.deinit(); - - var tree = try p.parse("[]"); - defer tree.deinit(); +test "Value.array allocator should still be usable after parsing" { + var parsed = try parseFromSlice(Value, std.testing.allocator, "[]", .{}); + defer parsed.deinit(); // Allocation should succeed var i: usize = 0; while (i < 100) : (i += 1) { - try tree.root.array.append(Value{ .integer = 100 }); + try parsed.value.array.append(Value{ .integer = 100 }); } - try testing.expectEqual(tree.root.array.items.len, 100); + try testing.expectEqual(parsed.value.array.items.len, 100); } test "integer after float has proper type" { @@ -184,45 +180,6 @@ test "escaped characters" { try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.string, "😂"); } -test "string copy option" { - const input = - \\{ - \\ "noescape": "aą😂", - \\ "simple": "\\\/\n\r\t\f\b\"", - \\ "unicode": "\u0105", - \\ "surrogatepair": "\ud83d\ude02" - \\} - ; - - var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_allocator.deinit(); - const allocator = arena_allocator.allocator(); - - var parser = Parser.init(allocator, .alloc_if_needed); - const tree_nocopy = try parser.parse(input); - const obj_nocopy = tree_nocopy.root.object; - - parser = Parser.init(allocator, .alloc_always); - const tree_copy = try parser.parse(input); - const obj_copy = tree_copy.root.object; - - for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) |field_name| { - try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.string, obj_copy.get(field_name).?.string); - } - - const nocopy_addr = &obj_nocopy.get("noescape").?.string[0]; - const copy_addr = &obj_copy.get("noescape").?.string[0]; - - var found_nocopy = false; - for (input, 0..) |_, index| { - try testing.expect(copy_addr != &input[index]); - if (nocopy_addr == &input[index]) { - found_nocopy = true; - } - } - try testing.expect(found_nocopy); -} - test "Value.jsonStringify" { { var buffer: [10]u8 = undefined; diff --git a/lib/std/json/static.zig b/lib/std/json/static.zig index 1b1692c380..73a209ebc0 100644 --- a/lib/std/json/static.zig +++ b/lib/std/json/static.zig @@ -1,6 +1,7 @@ const std = @import("std"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; const ArrayList = std.ArrayList; const Scanner = @import("./scanner.zig").Scanner; @@ -27,27 +28,79 @@ pub const ParseOptions = struct { max_value_len: ?usize = null, }; -/// Parses the json document from s and returns the result. -/// The provided allocator is used both for temporary allocations during parsing the document, -/// and also to allocate any pointer values in the return type. -/// If T contains any pointers, free the memory with `std.json.parseFree`. +pub fn Parsed(comptime T: type) type { + return struct { + arena: *ArenaAllocator, + value: T, + + pub fn deinit(self: @This()) void { + const allocator = self.arena.child_allocator; + self.arena.deinit(); + allocator.destroy(self.arena); + } + }; +} + +/// Parses the json document from `s` and returns the result packaged in a `std.json.Parsed`. +/// You must call `deinit()` of the returned object to clean up allocated resources. /// Note that `error.BufferUnderrun` is not actually possible to return from this function. -pub fn parseFromSlice(comptime T: type, allocator: Allocator, s: []const u8, options: ParseOptions) ParseError(T, Scanner)!T { +pub fn parseFromSlice( + comptime T: type, + allocator: Allocator, + s: []const u8, + options: ParseOptions, +) ParseError(Scanner)!Parsed(T) { var scanner = Scanner.initCompleteInput(allocator, s); defer scanner.deinit(); return parseFromTokenSource(T, allocator, &scanner, options); } +/// Parses the json document from `s` and returns the result. +/// Allocations made during this operation are not carefully tracked and may not be possible to individually clean up. +/// It is recommended to use a `std.heap.ArenaAllocator` or similar. +pub fn parseFromSliceLeaky( + comptime T: type, + allocator: Allocator, + s: []const u8, + options: ParseOptions, +) ParseError(Scanner)!T { + var scanner = Scanner.initCompleteInput(allocator, s); + defer scanner.deinit(); + + return parseFromTokenSourceLeaky(T, allocator, &scanner, options); +} + /// `scanner_or_reader` must be either a `*std.json.Scanner` with complete input or a `*std.json.Reader`. -/// allocator is used to allocate the data of T if necessary, -/// such as if T is `*u32` or `[]u32`. -/// If T contains any pointers, free the memory with `std.json.parseFree`. -/// If T contains no pointers, the allocator may sometimes be used for temporary allocations, -/// but no call to `std.json.parseFree` will be necessary; -/// all temporary allocations will be freed before this function returns. /// Note that `error.BufferUnderrun` is not actually possible to return from this function. -pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_reader: anytype, options: ParseOptions) ParseError(T, @TypeOf(scanner_or_reader.*))!T { +pub fn parseFromTokenSource( + comptime T: type, + allocator: Allocator, + scanner_or_reader: anytype, + options: ParseOptions, +) ParseError(@TypeOf(scanner_or_reader.*))!Parsed(T) { + var parsed = Parsed(T){ + .arena = try allocator.create(ArenaAllocator), + .value = undefined, + }; + errdefer allocator.destroy(parsed.arena); + parsed.arena.* = ArenaAllocator.init(allocator); + errdefer parsed.arena.deinit(); + + parsed.value = try parseFromTokenSourceLeaky(T, parsed.arena.allocator(), scanner_or_reader, options); + + return parsed; +} + +/// `scanner_or_reader` must be either a `*std.json.Scanner` with complete input or a `*std.json.Reader`. +/// Allocations made during this operation are not carefully tracked and may not be possible to individually clean up. +/// It is recommended to use a `std.heap.ArenaAllocator` or similar. +pub fn parseFromTokenSourceLeaky( + comptime T: type, + allocator: Allocator, + scanner_or_reader: anytype, + options: ParseOptions, +) ParseError(@TypeOf(scanner_or_reader.*))!T { if (@TypeOf(scanner_or_reader.*) == Scanner) { assert(scanner_or_reader.is_end_of_input); } @@ -61,80 +114,30 @@ pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_r } } - const r = try parseInternal(T, allocator, scanner_or_reader, resolved_options); - errdefer parseFree(T, allocator, r); + const value = try parseInternal(T, allocator, scanner_or_reader, resolved_options); assert(.end_of_document == try scanner_or_reader.next()); - return r; + return value; } -/// The error set that will be returned from parsing T from *Source. -/// Note that this may contain error.BufferUnderrun, but that error will never actually be returned. -pub fn ParseError(comptime T: type, comptime Source: type) type { - // `inferred_types` is used to avoid infinite recursion for recursive type definitions. - const inferred_types = [_]type{}; +/// The error set that will be returned when parsing from `*Source`. +/// Note that this may contain `error.BufferUnderrun`, but that error will never actually be returned. +pub fn ParseError(comptime Source: type) type { // A few of these will either always be present or present enough of the time that // omitting them is more confusing than always including them. - return error{UnexpectedToken} || Source.NextError || Source.PeekError || - ParseInternalErrorImpl(T, Source, &inferred_types); -} - -fn ParseInternalErrorImpl(comptime T: type, comptime Source: type, comptime inferred_types: []const type) type { - for (inferred_types) |ty| { - if (T == ty) return error{}; - } - - switch (@typeInfo(T)) { - .Bool => return error{}, - .Float, .ComptimeFloat => return Source.AllocError || std.fmt.ParseFloatError, - .Int, .ComptimeInt => { - return Source.AllocError || error{ InvalidNumber, Overflow } || - std.fmt.ParseIntError || std.fmt.ParseFloatError; - }, - .Optional => |optional_info| return ParseInternalErrorImpl(optional_info.child, Source, inferred_types ++ [_]type{T}), - .Enum => return Source.AllocError || error{InvalidEnumTag}, - .Union => |unionInfo| { - if (unionInfo.tag_type) |_| { - var errors = Source.AllocError || error{UnknownField}; - for (unionInfo.fields) |u_field| { - errors = errors || ParseInternalErrorImpl(u_field.type, Source, inferred_types ++ [_]type{T}); - } - return errors; - } else { - @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'"); - } - }, - .Struct => |structInfo| { - var errors = Scanner.AllocError || error{ - DuplicateField, - UnknownField, - MissingField, - }; - for (structInfo.fields) |field| { - errors = errors || ParseInternalErrorImpl(field.type, Source, inferred_types ++ [_]type{T}); - } - return errors; - }, - .Array => |arrayInfo| { - return error{LengthMismatch} || - ParseInternalErrorImpl(arrayInfo.child, Source, inferred_types ++ [_]type{T}); - }, - .Vector => |vecInfo| { - return error{LengthMismatch} || - ParseInternalErrorImpl(vecInfo.child, Source, inferred_types ++ [_]type{T}); - }, - .Pointer => |ptrInfo| { - switch (ptrInfo.size) { - .One, .Slice => { - return ParseInternalErrorImpl(ptrInfo.child, Source, inferred_types ++ [_]type{T}); - }, - else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"), - } - }, - else => return error{}, - } - unreachable; + return error{ + UnexpectedToken, + InvalidNumber, + Overflow, + InvalidEnumTag, + DuplicateField, + UnknownField, + MissingField, + LengthMismatch, + } || + std.fmt.ParseIntError || std.fmt.ParseFloatError || + Source.NextError || Source.PeekError || Source.AllocError; } fn parseInternal( @@ -142,7 +145,7 @@ fn parseInternal( allocator: Allocator, source: anytype, options: ParseOptions, -) ParseError(T, @TypeOf(source.*))!T { +) ParseError(@TypeOf(source.*))!T { switch (@typeInfo(T)) { .Bool => { return switch (try source.next()) { @@ -155,8 +158,7 @@ fn parseInternal( const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?); defer freeAllocated(allocator, token); const slice = switch (token) { - .number, .string => |slice| slice, - .allocated_number, .allocated_string => |slice| slice, + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; return try std.fmt.parseFloat(T, slice); @@ -165,8 +167,7 @@ fn parseInternal( const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?); defer freeAllocated(allocator, token); const slice = switch (token) { - .number, .string => |slice| slice, - .allocated_number, .allocated_string => |slice| slice, + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; if (isNumberFormattedLikeAnInteger(slice)) @@ -189,11 +190,14 @@ fn parseInternal( } }, .Enum => |enumInfo| { + if (comptime std.meta.trait.hasFn("jsonParse")(T)) { + return T.jsonParse(allocator, source, options); + } + const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?); defer freeAllocated(allocator, token); const slice = switch (token) { - .number, .string => |slice| slice, - .allocated_number, .allocated_string => |slice| slice, + inline .number, .allocated_number, .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; // Check for a named value. @@ -204,30 +208,18 @@ fn parseInternal( return try std.meta.intToEnum(T, n); }, .Union => |unionInfo| { - const UnionTagType = unionInfo.tag_type orelse @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'"); + if (comptime std.meta.trait.hasFn("jsonParse")(T)) { + return T.jsonParse(allocator, source, options); + } + + if (unionInfo.tag_type == null) @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'"); if (.object_begin != try source.next()) return error.UnexpectedToken; var result: ?T = null; - errdefer { - if (result) |r| { - inline for (unionInfo.fields) |u_field| { - if (r == @field(UnionTagType, u_field.name)) { - parseFree(u_field.type, allocator, @field(r, u_field.name)); - } - } - } - } - var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?); - errdefer { - if (name_token) |t| { - freeAllocated(allocator, t); - } - } const field_name = switch (name_token.?) { - .string => |slice| slice, - .allocated_string => |slice| slice, + inline .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; @@ -265,13 +257,6 @@ fn parseInternal( var r: T = undefined; var fields_seen: usize = 0; - errdefer { - inline for (0..structInfo.fields.len) |i| { - if (i < fields_seen) { - parseFree(structInfo.fields[i].type, allocator, r[i]); - } - } - } inline for (0..structInfo.fields.len) |i| { r[i] = try parseInternal(structInfo.fields[i].type, allocator, source, options); fields_seen = i + 1; @@ -282,29 +267,20 @@ fn parseInternal( return r; } + if (comptime std.meta.trait.hasFn("jsonParse")(T)) { + return T.jsonParse(allocator, source, options); + } + if (.object_begin != try source.next()) return error.UnexpectedToken; var r: T = undefined; var fields_seen = [_]bool{false} ** structInfo.fields.len; - errdefer { - inline for (structInfo.fields, 0..) |field, i| { - if (fields_seen[i]) { - parseFree(field.type, allocator, @field(r, field.name)); - } - } - } while (true) { var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?); - errdefer { - if (name_token) |t| { - freeAllocated(allocator, t); - } - } const field_name = switch (name_token.?) { .object_end => break, // No more fields. - .string => |slice| slice, - .allocated_string => |slice| slice, + inline .string, .allocated_string => |slice| slice, else => return error.UnexpectedToken, }; @@ -319,18 +295,13 @@ fn parseInternal( if (fields_seen[i]) { switch (options.duplicate_field_behavior) { .use_first => { - // Parse and then delete the redundant value. + // Parse and ignore the redundant value. // We don't want to skip the value, because we want type checking. - const ignored_value = try parseInternal(field.type, allocator, source, options); - parseFree(field.type, allocator, ignored_value); + _ = try parseInternal(field.type, allocator, source, options); break; }, .@"error" => return error.DuplicateField, - .use_last => { - // Delete the stale value. We're about to get a new one. - parseFree(field.type, allocator, @field(r, field.name)); - fields_seen[i] = false; - }, + .use_last => {}, } } @field(r, field.name) = try parseInternal(field.type, allocator, source, options); @@ -428,7 +399,6 @@ fn parseInternal( switch (ptrInfo.size) { .One => { const r: *ptrInfo.child = try allocator.create(ptrInfo.child); - errdefer allocator.destroy(r); r.* = try parseInternal(ptrInfo.child, allocator, source, options); return r; }, @@ -439,13 +409,6 @@ fn parseInternal( // Typical array. var arraylist = ArrayList(ptrInfo.child).init(allocator); - errdefer { - while (arraylist.popOrNull()) |v| { - parseFree(ptrInfo.child, allocator, v); - } - arraylist.deinit(); - } - while (true) { switch (try source.peekNextTokenType()) { .array_end => { @@ -473,13 +436,20 @@ fn parseInternal( if (ptrInfo.sentinel) |sentinel_ptr| { // Use our own array list so we can append the sentinel. var value_list = ArrayList(u8).init(allocator); - errdefer value_list.deinit(); _ = try source.allocNextIntoArrayList(&value_list, .alloc_always); return try value_list.toOwnedSliceSentinel(@ptrCast(*const u8, sentinel_ptr).*); } - switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) { - .allocated_string => |slice| return slice, - else => unreachable, + if (ptrInfo.is_const) { + switch (try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?)) { + inline .string, .allocated_string => |slice| return slice, + else => unreachable, + } + } else { + // Have to allocate to get a mutable copy. + switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) { + .allocated_string => |slice| return slice, + else => unreachable, + } } }, else => return error.UnexpectedToken, @@ -505,13 +475,6 @@ fn parseInternalArray( var r: T = undefined; var i: usize = 0; - errdefer { - // Without the len check `r[i]` is not allowed - if (len > 0) while (true) : (i -= 1) { - parseFree(Child, allocator, r[i]); - if (i == 0) break; - }; - } while (i < len) : (i += 1) { r[i] = try parseInternal(Child, allocator, source, options); } @@ -530,92 +493,6 @@ fn freeAllocated(allocator: Allocator, token: Token) void { } } -/// Releases resources created by parseFromSlice() or parseFromTokenSource(). -pub fn parseFree(comptime T: type, allocator: Allocator, value: T) void { - switch (@typeInfo(T)) { - .Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {}, - .Optional => { - if (value) |v| { - return parseFree(@TypeOf(v), allocator, v); - } - }, - .Union => |unionInfo| { - if (unionInfo.tag_type) |UnionTagType| { - inline for (unionInfo.fields) |u_field| { - if (value == @field(UnionTagType, u_field.name)) { - parseFree(u_field.type, allocator, @field(value, u_field.name)); - break; - } - } - } else { - unreachable; - } - }, - .Struct => |structInfo| { - inline for (structInfo.fields) |field| { - var should_free = true; - if (field.default_value) |default| { - switch (@typeInfo(field.type)) { - // We must not attempt to free pointers to struct default values - .Pointer => |fieldPtrInfo| { - const field_value = @field(value, field.name); - const field_ptr = switch (fieldPtrInfo.size) { - .One => field_value, - .Slice => field_value.ptr, - else => unreachable, // Other pointer types are not parseable - }; - const field_addr = @ptrToInt(field_ptr); - - const casted_default = @ptrCast(*const field.type, @alignCast(@alignOf(field.type), default)).*; - const default_ptr = switch (fieldPtrInfo.size) { - .One => casted_default, - .Slice => casted_default.ptr, - else => unreachable, // Other pointer types are not parseable - }; - const default_addr = @ptrToInt(default_ptr); - - if (field_addr == default_addr) { - should_free = false; - } - }, - else => {}, - } - } - if (should_free) { - parseFree(field.type, allocator, @field(value, field.name)); - } - } - }, - .Array => |arrayInfo| { - for (value) |v| { - parseFree(arrayInfo.child, allocator, v); - } - }, - .Vector => |vecInfo| { - var i: usize = 0; - while (i < vecInfo.len) : (i += 1) { - parseFree(vecInfo.child, allocator, value[i]); - } - }, - .Pointer => |ptrInfo| { - switch (ptrInfo.size) { - .One => { - parseFree(ptrInfo.child, allocator, value.*); - allocator.destroy(value); - }, - .Slice => { - for (value) |v| { - parseFree(ptrInfo.child, allocator, v); - } - allocator.free(value); - }, - else => unreachable, - } - }, - else => unreachable, - } -} - test { _ = @import("./static_test.zig"); } diff --git a/lib/std/json/static_test.zig b/lib/std/json/static_test.zig index b512f8a890..c4e2ab8d2d 100644 --- a/lib/std/json/static_test.zig +++ b/lib/std/json/static_test.zig @@ -1,29 +1,31 @@ const std = @import("std"); const testing = std.testing; +const ArenaAllocator = std.heap.ArenaAllocator; const parseFromSlice = @import("./static.zig").parseFromSlice; +const parseFromSliceLeaky = @import("./static.zig").parseFromSliceLeaky; const parseFromTokenSource = @import("./static.zig").parseFromTokenSource; -const parseFree = @import("./static.zig").parseFree; +const parseFromTokenSourceLeaky = @import("./static.zig").parseFromTokenSourceLeaky; const ParseOptions = @import("./static.zig").ParseOptions; const JsonScanner = @import("./scanner.zig").Scanner; const jsonReader = @import("./scanner.zig").reader; test "parse" { - try testing.expectEqual(false, try parseFromSlice(bool, testing.allocator, "false", .{})); - try testing.expectEqual(true, try parseFromSlice(bool, testing.allocator, "true", .{})); - try testing.expectEqual(@as(u1, 1), try parseFromSlice(u1, testing.allocator, "1", .{})); - try testing.expectError(error.Overflow, parseFromSlice(u1, testing.allocator, "50", .{})); - try testing.expectEqual(@as(u64, 42), try parseFromSlice(u64, testing.allocator, "42", .{})); - try testing.expectEqual(@as(f64, 42), try parseFromSlice(f64, testing.allocator, "42.0", .{})); - try testing.expectEqual(@as(?bool, null), try parseFromSlice(?bool, testing.allocator, "null", .{})); - try testing.expectEqual(@as(?bool, true), try parseFromSlice(?bool, testing.allocator, "true", .{})); + try testing.expectEqual(false, try parseFromSliceLeaky(bool, testing.allocator, "false", .{})); + try testing.expectEqual(true, try parseFromSliceLeaky(bool, testing.allocator, "true", .{})); + try testing.expectEqual(@as(u1, 1), try parseFromSliceLeaky(u1, testing.allocator, "1", .{})); + try testing.expectError(error.Overflow, parseFromSliceLeaky(u1, testing.allocator, "50", .{})); + try testing.expectEqual(@as(u64, 42), try parseFromSliceLeaky(u64, testing.allocator, "42", .{})); + try testing.expectEqual(@as(f64, 42), try parseFromSliceLeaky(f64, testing.allocator, "42.0", .{})); + try testing.expectEqual(@as(?bool, null), try parseFromSliceLeaky(?bool, testing.allocator, "null", .{})); + try testing.expectEqual(@as(?bool, true), try parseFromSliceLeaky(?bool, testing.allocator, "true", .{})); - try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "\"foo\"", .{})); - try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "[102, 111, 111]", .{})); - try testing.expectEqual(@as([0]u8, undefined), try parseFromSlice([0]u8, testing.allocator, "[]", .{})); + try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSliceLeaky([3]u8, testing.allocator, "\"foo\"", .{})); + try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSliceLeaky([3]u8, testing.allocator, "[102, 111, 111]", .{})); + try testing.expectEqual(@as([0]u8, undefined), try parseFromSliceLeaky([0]u8, testing.allocator, "[]", .{})); - try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSlice(u64, testing.allocator, "\"12345678901234567890\"", .{})); - try testing.expectEqual(@as(f64, 123.456), try parseFromSlice(f64, testing.allocator, "\"123.456\"", .{})); + try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSliceLeaky(u64, testing.allocator, "\"12345678901234567890\"", .{})); + try testing.expectEqual(@as(f64, 123.456), try parseFromSliceLeaky(f64, testing.allocator, "\"123.456\"", .{})); } test "parse into enum" { @@ -32,37 +34,37 @@ test "parse into enum" { Bar, @"with\\escape", }; - try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "\"Foo\"", .{})); - try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "42", .{})); - try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSlice(T, testing.allocator, "\"with\\\\escape\"", .{})); - try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "5", .{})); - try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "\"Qux\"", .{})); + try testing.expectEqual(@as(T, .Foo), try parseFromSliceLeaky(T, testing.allocator, "\"Foo\"", .{})); + try testing.expectEqual(@as(T, .Foo), try parseFromSliceLeaky(T, testing.allocator, "42", .{})); + try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSliceLeaky(T, testing.allocator, "\"with\\\\escape\"", .{})); + try testing.expectError(error.InvalidEnumTag, parseFromSliceLeaky(T, testing.allocator, "5", .{})); + try testing.expectError(error.InvalidEnumTag, parseFromSliceLeaky(T, testing.allocator, "\"Qux\"", .{})); } test "parse into that allocates a slice" { { // string as string - const r = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{}); - defer parseFree([]u8, testing.allocator, r); - try testing.expectEqualSlices(u8, "foo", r); + const parsed = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{}); + defer parsed.deinit(); + try testing.expectEqualSlices(u8, "foo", parsed.value); } { // string as array of u8 integers - const r = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{}); - defer parseFree([]u8, testing.allocator, r); - try testing.expectEqualSlices(u8, "foo", r); + const parsed = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{}); + defer parsed.deinit(); + try testing.expectEqualSlices(u8, "foo", parsed.value); } { - const r = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{}); - defer parseFree([]u8, testing.allocator, r); - try testing.expectEqualSlices(u8, "with\\escape", r); + const parsed = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{}); + defer parsed.deinit(); + try testing.expectEqualSlices(u8, "with\\escape", parsed.value); } } test "parse into sentinel slice" { - const result = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{}); - defer parseFree([:0]const u8, testing.allocator, result); - try testing.expect(std.mem.eql(u8, result, "\n")); + const parsed = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{}); + defer parsed.deinit(); + try testing.expect(std.mem.eql(u8, parsed.value, "\n")); } test "parse into tagged union" { @@ -72,9 +74,12 @@ test "parse into tagged union" { float: f64, string: []const u8, }; - try testing.expectEqual(T{ .float = 1.5 }, try parseFromSlice(T, testing.allocator, "{\"float\":1.5}", .{})); - try testing.expectEqual(T{ .int = 1 }, try parseFromSlice(T, testing.allocator, "{\"int\":1}", .{})); - try testing.expectEqual(T{ .nothing = {} }, try parseFromSlice(T, testing.allocator, "{\"nothing\":{}}", .{})); + try testing.expectEqual(T{ .float = 1.5 }, try parseFromSliceLeaky(T, testing.allocator, "{\"float\":1.5}", .{})); + try testing.expectEqual(T{ .int = 1 }, try parseFromSliceLeaky(T, testing.allocator, "{\"int\":1}", .{})); + try testing.expectEqual(T{ .nothing = {} }, try parseFromSliceLeaky(T, testing.allocator, "{\"nothing\":{}}", .{})); + const parsed = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{}); + defer parsed.deinit(); + try testing.expectEqualSlices(u8, "foo", parsed.value.string); } test "parse into tagged union errors" { @@ -84,42 +89,36 @@ test "parse into tagged union errors" { float: f64, string: []const u8, }; - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "42", .{})); - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{}", .{})); - try testing.expectError(error.UnknownField, parseFromSlice(T, testing.allocator, "{\"bogus\":1}", .{})); - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"int\":1", .{})); - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"float\":1.0}", .{})); - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":null}", .{})); - try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":{\"no\":0}}", .{})); + var arena = ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "42", .{})); + try testing.expectError(error.SyntaxError, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1} 42", .{})); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{}", .{})); + try testing.expectError(error.UnknownField, parseFromSliceLeaky(T, arena.allocator(), "{\"bogus\":1}", .{})); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1, \"int\":1", .{})); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1, \"float\":1.0}", .{})); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"nothing\":null}", .{})); + try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"nothing\":{\"no\":0}}", .{})); // Allocator failure var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0); - const failing_allocator = fail_alloc.allocator(); - try testing.expectError(error.OutOfMemory, parseFromSlice(T, failing_allocator, "{\"string\"\"foo\"}", .{})); -} - -test "parseFree descends into tagged union" { - const T = union(enum) { - nothing, - int: i32, - float: f64, - string: []const u8, - }; - const r = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{}); - try testing.expectEqualSlices(u8, "foo", r.string); - parseFree(T, testing.allocator, r); + try testing.expectError(error.OutOfMemory, parseFromSlice(T, fail_alloc.allocator(), "{\"string\"\"foo\"}", .{})); } test "parse into struct with no fields" { const T = struct {}; - try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{})); + const parsed = try parseFromSlice(T, testing.allocator, "{}", .{}); + defer parsed.deinit(); + try testing.expectEqual(T{}, parsed.value); } const test_const_value: usize = 123; test "parse into struct with default const pointer field" { const T = struct { a: *const usize = &test_const_value }; - try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{})); + const parsed = try parseFromSlice(T, testing.allocator, "{}", .{}); + defer parsed.deinit(); + try testing.expectEqual(T{}, parsed.value); } const test_default_usize: usize = 123; @@ -138,10 +137,9 @@ test "freeing parsed structs with pointers to default values" { str_slice: []const []const u8 = &test_default_str_slice, }; - const parsed = try parseFromSlice(T, testing.allocator, "{}", .{}); - try testing.expectEqual(T{}, parsed); - // This will panic if it tries to free global constants: - parseFree(T, testing.allocator, parsed); + var parsed = try parseFromSlice(T, testing.allocator, "{}", .{}); + try testing.expectEqual(T{}, parsed.value); + defer parsed.deinit(); } test "parse into struct where destination and source lengths mismatch" { @@ -201,8 +199,9 @@ test "parse into struct with misc fields" { \\ } \\} ; - const r = try parseFromSlice(T, testing.allocator, document_str, .{}); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, document_str, .{}); + defer parsed.deinit(); + const r = &parsed.value; try testing.expectEqual(@as(i64, 420), r.int); try testing.expectEqual(@as(f64, 3.14), r.float); try testing.expectEqual(true, r.@"with\\escape"); @@ -238,24 +237,20 @@ test "parse into struct with strings and arrays with sentinels" { \\ "simple_data": [4, 5, 6] \\} ; - const r = try parseFromSlice(T, testing.allocator, document_str, .{}); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, document_str, .{}); + defer parsed.deinit(); - try testing.expectEqualSentinel(u8, 0, "zig", r.language); + try testing.expectEqualSentinel(u8, 0, "zig", parsed.value.language); const data = [_:99]i32{ 1, 2, 3 }; - try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data); + try testing.expectEqualSentinel(i32, 99, data[0..data.len], parsed.value.data); // Make sure that arrays who aren't supposed to have a sentinel still parse without one. - try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data))); - try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel))); + try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(parsed.value.simple_data))); + try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(parsed.value.language_without_sentinel))); } test "parse into struct with duplicate field" { - // allow allocator to detect double frees by keeping bucket in use - const ballast = try testing.allocator.alloc(u64, 1); - defer testing.allocator.free(ballast); - const options_first = ParseOptions{ .duplicate_field_behavior = .use_first }; const options_last = ParseOptions{ .duplicate_field_behavior = .use_last }; @@ -266,9 +261,12 @@ test "parse into struct with duplicate field" { try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_first)); try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_last)); + var arena = ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const T2 = struct { a: f64 }; - try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSlice(T2, testing.allocator, str, options_first)); - try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSlice(T2, testing.allocator, str, options_last)); + try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSliceLeaky(T2, arena.allocator(), str, options_first)); + try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSliceLeaky(T2, arena.allocator(), str, options_last)); } test "parse into struct ignoring unknown fields" { @@ -302,11 +300,11 @@ test "parse into struct ignoring unknown fields" { \\ "language": "zig" \\} ; - const r = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true }); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true }); + defer parsed.deinit(); - try testing.expectEqual(@as(i64, 420), r.int); - try testing.expectEqualSlices(u8, "zig", r.language); + try testing.expectEqual(@as(i64, 420), parsed.value.int); + try testing.expectEqualSlices(u8, "zig", parsed.value.language); } test "parse into tuple" { @@ -343,8 +341,9 @@ test "parse into tuple" { \\ {"float": 12.34} \\] ; - const r = try parseFromSlice(T, testing.allocator, str, .{}); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, str, .{}); + defer parsed.deinit(); + const r = parsed.value; try testing.expectEqual(@as(i64, 420), r[0]); try testing.expectEqual(@as(f64, 3.14), r[1]); try testing.expectEqual(true, r[2]); @@ -368,10 +367,10 @@ test "parse into recursive union definition" { values: ParseIntoRecursiveUnionDefinitionValue, }; - const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{}); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{}); + defer parsed.deinit(); - try testing.expectEqual(@as(i64, 58), r.values.array[0].integer); + try testing.expectEqual(@as(i64, 58), parsed.value.values.array[0].integer); } const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) { @@ -389,29 +388,37 @@ test "parse into double recursive union definition" { values: ParseIntoDoubleRecursiveUnionValueFirst, }; - const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{}); - defer parseFree(T, testing.allocator, r); + const parsed = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{}); + defer parsed.deinit(); - try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer); + try testing.expectEqual(@as(i64, 58), parsed.value.values.array[0].array[0].integer); } test "parse exponential into int" { const T = struct { int: i64 }; - const r = try parseFromSlice(T, testing.allocator, "{ \"int\": 4.2e2 }", .{}); + const r = try parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 4.2e2 }", .{}); try testing.expectEqual(@as(i64, 420), r.int); - try testing.expectError(error.InvalidNumber, parseFromSlice(T, testing.allocator, "{ \"int\": 0.042e2 }", .{})); - try testing.expectError(error.Overflow, parseFromSlice(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{})); + try testing.expectError(error.InvalidNumber, parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 0.042e2 }", .{})); + try testing.expectError(error.Overflow, parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{})); } test "parseFromTokenSource" { - var scanner = JsonScanner.initCompleteInput(testing.allocator, "123"); - defer scanner.deinit(); - try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &scanner, .{})); + { + var scanner = JsonScanner.initCompleteInput(testing.allocator, "123"); + defer scanner.deinit(); + var parsed = try parseFromTokenSource(u32, testing.allocator, &scanner, .{}); + defer parsed.deinit(); + try testing.expectEqual(@as(u32, 123), parsed.value); + } - var stream = std.io.fixedBufferStream("123"); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); - defer json_reader.deinit(); - try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &json_reader, .{})); + { + var stream = std.io.fixedBufferStream("123"); + var json_reader = jsonReader(std.testing.allocator, stream.reader()); + defer json_reader.deinit(); + var parsed = try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}); + defer parsed.deinit(); + try testing.expectEqual(@as(u32, 123), parsed.value); + } } test "max_value_len" { @@ -429,9 +436,9 @@ test "parse into vector" { \\ "vec_i32": [4, 5, 6, 7] \\} ; - const r = try parseFromSlice(T, testing.allocator, s, .{}); - defer parseFree(T, testing.allocator, r); - try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001); - try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001); - try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32); + const parsed = try parseFromSlice(T, testing.allocator, s, .{}); + defer parsed.deinit(); + try testing.expectApproxEqAbs(@as(f32, 1.5), parsed.value.vec_f32[0], 0.0000001); + try testing.expectApproxEqAbs(@as(f32, 2.5), parsed.value.vec_f32[1], 0.0000001); + try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, parsed.value.vec_i32); } diff --git a/lib/std/json/test.zig b/lib/std/json/test.zig index 7135bc1b34..939f2e194c 100644 --- a/lib/std/json/test.zig +++ b/lib/std/json/test.zig @@ -1,8 +1,9 @@ const std = @import("std"); const testing = std.testing; -const Parser = @import("./dynamic.zig").Parser; +const parseFromSlice = @import("./static.zig").parseFromSlice; const validate = @import("./scanner.zig").validate; const JsonScanner = @import("./scanner.zig").Scanner; +const Value = @import("./dynamic.zig").Value; // Support for JSONTestSuite.zig pub fn ok(s: []const u8) !void { @@ -26,10 +27,8 @@ fn testLowLevelScanner(s: []const u8) !void { } } fn testHighLevelDynamicParser(s: []const u8) !void { - var p = Parser.init(testing.allocator, .alloc_if_needed); - defer p.deinit(); - var tree = try p.parse(s); - defer tree.deinit(); + var parsed = try parseFromSlice(Value, testing.allocator, s, .{}); + defer parsed.deinit(); } // Additional tests not part of test JSONTestSuite. @@ -47,15 +46,12 @@ test "n_object_closed_missing_value" { fn roundTrip(s: []const u8) !void { try testing.expect(try validate(testing.allocator, s)); - var p = Parser.init(testing.allocator, .alloc_if_needed); - defer p.deinit(); - - var tree = try p.parse(s); - defer tree.deinit(); + var parsed = try parseFromSlice(Value, testing.allocator, s, .{}); + defer parsed.deinit(); var buf: [256]u8 = undefined; var fbs = std.io.fixedBufferStream(&buf); - try tree.root.jsonStringify(.{}, fbs.writer()); + try parsed.value.jsonStringify(.{}, fbs.writer()); try testing.expectEqualStrings(s, fbs.getWritten()); } diff --git a/tools/gen_spirv_spec.zig b/tools/gen_spirv_spec.zig index b48e3834a2..91d0ba80ac 100644 --- a/tools/gen_spirv_spec.zig +++ b/tools/gen_spirv_spec.zig @@ -20,15 +20,16 @@ pub fn main() !void { // Required for json parsing. @setEvalBranchQuota(10000); - var registry = try std.json.parseFromSlice(g.Registry, allocator, spec, .{}); - - const core_reg = switch (registry) { - .core => |core_reg| core_reg, - .extension => return error.TODOSpirVExtensionSpec, + var scanner = std.json.Scanner.initCompleteInput(allocator, spec); + var diagnostics = std.json.Diagnostics{}; + scanner.enableDiagnostics(&diagnostics); + var parsed = std.json.parseFromTokenSource(g.CoreRegistry, allocator, &scanner, .{}) catch |err| { + std.debug.print("line,col: {},{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() }); + return err; }; var bw = std.io.bufferedWriter(std.io.getStdOut().writer()); - try render(bw.writer(), allocator, core_reg); + try render(bw.writer(), allocator, parsed.value); try bw.flush(); } diff --git a/tools/spirv/grammar.zig b/tools/spirv/grammar.zig index 86a01641e4..446ed0a062 100644 --- a/tools/spirv/grammar.zig +++ b/tools/spirv/grammar.zig @@ -1,6 +1,9 @@ //! See https://www.khronos.org/registry/spir-v/specs/unified1/MachineReadableGrammar.html //! and the files in https://github.com/KhronosGroup/SPIRV-Headers/blob/master/include/spirv/unified1/ //! Note: Non-canonical casing in these structs used to match SPIR-V spec json. + +const std = @import("std"); + pub const Registry = union(enum) { core: CoreRegistry, extension: ExtensionRegistry, @@ -79,6 +82,20 @@ pub const Enumerant = struct { value: union(enum) { bitflag: []const u8, // Hexadecimal representation of the value int: u31, + + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) std.json.ParseError(@TypeOf(source.*))!@This() { + _ = options; + switch (try source.nextAlloc(allocator, .alloc_if_needed)) { + inline .string, .allocated_string => |s| return @This(){ .bitflag = s }, + inline .number, .allocated_number => |s| return @This(){ .int = try std.fmt.parseInt(u31, s, 10) }, + else => return error.UnexpectedToken, + } + } + pub const jsonStringify = @compileError("not supported"); }, capabilities: [][]const u8 = &[_][]const u8{}, /// Valid for .ValueEnum and .BitEnum diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig index feefeb0a83..5d105195ef 100644 --- a/tools/update_clang_options.zig +++ b/tools/update_clang_options.zig @@ -624,9 +624,9 @@ pub fn main() anyerror!void { }, }; - var parser = json.Parser.init(allocator, .alloc_if_needed); - const tree = try parser.parse(json_text); - const root_map = &tree.root.object; + const parsed = try json.parseFromSlice(json.Value, allocator, json_text, .{}); + defer parsed.deinit(); + const root_map = &parsed.value.object; var all_objects = std.ArrayList(*json.ObjectMap).init(allocator); { diff --git a/tools/update_cpu_features.zig b/tools/update_cpu_features.zig index d5c3d48852..a4b9607a40 100644 --- a/tools/update_cpu_features.zig +++ b/tools/update_cpu_features.zig @@ -1054,14 +1054,14 @@ fn processOneTarget(job: Job) anyerror!void { var json_parse_progress = progress_node.start("parse JSON", 0); json_parse_progress.activate(); - var parser = json.Parser.init(arena, .alloc_if_needed); - const tree = try parser.parse(json_text); + const parsed = try json.parseFromSlice(json.Value, arena, json_text, .{}); + defer parsed.deinit(); + const root_map = &parsed.value.object; json_parse_progress.end(); var render_progress = progress_node.start("render zig code", 0); render_progress.activate(); - const root_map = &tree.root.object; var features_table = std.StringHashMap(Feature).init(arena); var all_features = std.ArrayList(Feature).init(arena); var all_cpus = std.ArrayList(Cpu).init(arena); diff --git a/tools/update_spirv_features.zig b/tools/update_spirv_features.zig index 6a923cf72a..6fd1b26c7f 100644 --- a/tools/update_spirv_features.zig +++ b/tools/update_spirv_features.zig @@ -74,7 +74,13 @@ pub fn main() !void { const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" }); const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize)); - const registry = try std.json.parseFromSlice(g.CoreRegistry, allocator, registry_json, .{}); + var scanner = std.json.Scanner.initCompleteInput(allocator, registry_json); + var diagnostics = std.json.Diagnostics{}; + scanner.enableDiagnostics(&diagnostics); + const registry = std.json.parseFromTokenSourceLeaky(g.CoreRegistry, allocator, &scanner, .{}) catch |err| { + std.debug.print("line,col: {},{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() }); + return err; + }; const capabilities = for (registry.operand_kinds) |opkind| { if (std.mem.eql(u8, opkind.kind, "Capability"))