mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2025-12-06 13:54:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

parser: make some errors point to end of previous token

Browse source 
For some errors if the found token is not on the same line as
the previous token, point to the end of the previous token.
This usually results in more helpful errors.
This commit is contained in:
Veikka Tuominen 2022-02-16 13:06:11 +02:00
parent 5283a52af5
commit 9c36cf92f0
8 changed files with 148 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
lib/std/zig/Ast.zig
View file

@ -66,20 +66,11 @@ pub fn renderToArrayList(tree: Ast, buffer: *std.ArrayList(u8)) RenderError!void
/// Returns an extra offset for column and byte offset of errors that /// Returns an extra offset for column and byte offset of errors that
/// should point after the token in the error message. /// should point after the token in the error message.
pub fn errorOffset(tree: Ast, error_tag: Error.Tag, token: TokenIndex) u32 { pub fn errorOffset(tree: Ast, parse_error: Error) u32 {
return switch (error_tag) { return if (parse_error.token_is_prev)
.expected_semi_after_decl, @intCast(u32, tree.tokenSlice(parse_error.token).len)
.expected_semi_after_stmt, else
.expected_comma_after_field, 0;
.expected_comma_after_arg,
.expected_comma_after_param,
.expected_comma_after_initializer,
.expected_comma_after_switch_prong,
.expected_semi_or_else,
.expected_semi_or_lbrace,
=> @intCast(u32, tree.tokenSlice(token).len),
else => 0,
};
} }
pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location { pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
@ -162,22 +153,22 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
}, },
.expected_block => { .expected_block => {
return stream.print("expected block or field, found '{s}'", .{ return stream.print("expected block or field, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_block_or_assignment => { .expected_block_or_assignment => {
return stream.print("expected block or assignment, found '{s}'", .{ return stream.print("expected block or assignment, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_block_or_expr => { .expected_block_or_expr => {
return stream.print("expected block or expression, found '{s}'", .{ return stream.print("expected block or expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_block_or_field => { .expected_block_or_field => {
return stream.print("expected block or field, found '{s}'", .{ return stream.print("expected block or field, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_container_members => { .expected_container_members => {
@ -187,42 +178,42 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
}, },
.expected_expr => { .expected_expr => {
return stream.print("expected expression, found '{s}'", .{ return stream.print("expected expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_expr_or_assignment => { .expected_expr_or_assignment => {
return stream.print("expected expression or assignment, found '{s}'", .{ return stream.print("expected expression or assignment, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_fn => { .expected_fn => {
return stream.print("expected function, found '{s}'", .{ return stream.print("expected function, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_inlinable => { .expected_inlinable => {
return stream.print("expected 'while' or 'for', found '{s}'", .{ return stream.print("expected 'while' or 'for', found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_labelable => { .expected_labelable => {
return stream.print("expected 'while', 'for', 'inline', 'suspend', or '{{', found '{s}'", .{ return stream.print("expected 'while', 'for', 'inline', 'suspend', or '{{', found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_param_list => { .expected_param_list => {
return stream.print("expected parameter list, found '{s}'", .{ return stream.print("expected parameter list, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_prefix_expr => { .expected_prefix_expr => {
return stream.print("expected prefix expression, found '{s}'", .{ return stream.print("expected prefix expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_primary_type_expr => { .expected_primary_type_expr => {
return stream.print("expected primary type expression, found '{s}'", .{ return stream.print("expected primary type expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_pub_item => { .expected_pub_item => {
@ -230,7 +221,7 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
}, },
.expected_return_type => { .expected_return_type => {
return stream.print("expected return type expression, found '{s}'", .{ return stream.print("expected return type expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_semi_or_else => { .expected_semi_or_else => {
@ -244,39 +235,34 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
token_tags[parse_error.token].symbol(), token_tags[parse_error.token].symbol(),
}); });
}, },
.expected_string_literal => {
return stream.print("expected string literal, found '{s}'", .{
token_tags[parse_error.token].symbol(),
});
},
.expected_suffix_op => { .expected_suffix_op => {
return stream.print("expected pointer dereference, optional unwrap, or field access, found '{s}'", .{ return stream.print("expected pointer dereference, optional unwrap, or field access, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_type_expr => { .expected_type_expr => {
return stream.print("expected type expression, found '{s}'", .{ return stream.print("expected type expression, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_var_decl => { .expected_var_decl => {
return stream.print("expected variable declaration, found '{s}'", .{ return stream.print("expected variable declaration, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_var_decl_or_fn => { .expected_var_decl_or_fn => {
return stream.print("expected variable declaration or function, found '{s}'", .{ return stream.print("expected variable declaration or function, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_loop_payload => { .expected_loop_payload => {
return stream.print("expected loop payload, found '{s}'", .{ return stream.print("expected loop payload, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.expected_container => { .expected_container => {
return stream.print("expected a struct, enum or union, found '{s}'", .{ return stream.print("expected a struct, enum or union, found '{s}'", .{
token_tags[parse_error.token].symbol(), token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)].symbol(),
}); });
}, },
.extern_fn_body => { .extern_fn_body => {
@ -341,9 +327,12 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
.expected_comma_after_switch_prong => { .expected_comma_after_switch_prong => {
return stream.writeAll("expected ',' after switch prong"); return stream.writeAll("expected ',' after switch prong");
}, },
.expected_initializer => {
return stream.writeAll("expected field initializer");
},
.expected_token => { .expected_token => {
const found_tag = token_tags[parse_error.token]; const found_tag = token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)];
const expected_symbol = parse_error.extra.expected_tag.symbol(); const expected_symbol = parse_error.extra.expected_tag.symbol();
switch (found_tag) { switch (found_tag) {
.invalid => return stream.print("expected '{s}', found invalid bytes", .{ .invalid => return stream.print("expected '{s}', found invalid bytes", .{
@ -2483,6 +2472,7 @@ pub const full = struct {
pub const Error = struct { pub const Error = struct {
tag: Tag, tag: Tag,
token_is_prev: bool = false,
token: TokenIndex, token: TokenIndex,
extra: union { extra: union {
none: void, none: void,
@ -2511,7 +2501,6 @@ pub const Error = struct {
expected_semi_or_else, expected_semi_or_else,
expected_semi_or_lbrace, expected_semi_or_lbrace,
expected_statement, expected_statement,
expected_string_literal,
expected_suffix_op, expected_suffix_op,
expected_type_expr, expected_type_expr,
expected_var_decl, expected_var_decl,
@ -2539,6 +2528,7 @@ pub const Error = struct {
expected_comma_after_param, expected_comma_after_param,
expected_comma_after_initializer, expected_comma_after_initializer,
expected_comma_after_switch_prong, expected_comma_after_switch_prong,
expected_initializer,
/// `expected_tag` is populated. /// `expected_tag` is populated.
expected_token, expected_token,

162
lib/std/zig/parse.zig
View file

@ -147,11 +147,6 @@ const Parser = struct {
return result; return result;
} }
fn warn(p: *Parser, tag: Ast.Error.Tag) error{OutOfMemory}!void {
@setCold(true);
try p.warnMsg(.{ .tag = tag, .token = p.tok_i });
}
fn warnExpected(p: *Parser, expected_token: Token.Tag) error{OutOfMemory}!void { fn warnExpected(p: *Parser, expected_token: Token.Tag) error{OutOfMemory}!void {
@setCold(true); @setCold(true);
try p.warnMsg(.{ try p.warnMsg(.{
@ -161,13 +156,53 @@ const Parser = struct {
}); });
} }
fn warnExpectedAfter(p: *Parser, error_tag: AstError.Tag) error{OutOfMemory}!void { fn warn(p: *Parser, error_tag: AstError.Tag) error{OutOfMemory}!void {
@setCold(true); @setCold(true);
try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i - 1 }); try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i });
} }
fn warnMsg(p: *Parser, msg: Ast.Error) error{OutOfMemory}!void { fn warnMsg(p: *Parser, msg: Ast.Error) error{OutOfMemory}!void {
@setCold(true); @setCold(true);
switch (msg.tag) {
.expected_semi_after_decl,
.expected_semi_after_stmt,
.expected_comma_after_field,
.expected_comma_after_arg,
.expected_comma_after_param,
.expected_comma_after_initializer,
.expected_comma_after_switch_prong,
.expected_semi_or_else,
.expected_semi_or_lbrace,
.expected_token,
.expected_block,
.expected_block_or_assignment,
.expected_block_or_expr,
.expected_block_or_field,
.expected_container_members,
.expected_expr,
.expected_expr_or_assignment,
.expected_fn,
.expected_inlinable,
.expected_labelable,
.expected_param_list,
.expected_prefix_expr,
.expected_primary_type_expr,
.expected_pub_item,
.expected_return_type,
.expected_suffix_op,
.expected_type_expr,
.expected_var_decl,
.expected_var_decl_or_fn,
.expected_loop_payload,
.expected_container,
=> if (msg.token != 0 and !p.tokensOnSameLine(msg.token - 1, msg.token)) {
var copy = msg;
copy.token_is_prev = true;
copy.token -= 1;
return p.errors.append(p.gpa, copy);
},
else => {},
}
try p.errors.append(p.gpa, msg); try p.errors.append(p.gpa, msg);
} }
@ -264,7 +299,7 @@ const Parser = struct {
} }
// There is not allowed to be a decl after a field with no comma. // There is not allowed to be a decl after a field with no comma.
// Report error but recover parser. // Report error but recover parser.
try p.warnExpectedAfter(.expected_comma_after_field); try p.warn(.expected_comma_after_field);
p.findNextContainerMember(); p.findNextContainerMember();
} }
}, },
@ -367,7 +402,7 @@ const Parser = struct {
} }
// There is not allowed to be a decl after a field with no comma. // There is not allowed to be a decl after a field with no comma.
// Report error but recover parser. // Report error but recover parser.
try p.warnExpectedAfter(.expected_comma_after_field); try p.warn(.expected_comma_after_field);
p.findNextContainerMember(); p.findNextContainerMember();
} }
}, },
@ -585,7 +620,7 @@ const Parser = struct {
// Since parseBlock only return error.ParseError on // Since parseBlock only return error.ParseError on
// a missing '}' we can assume this function was // a missing '}' we can assume this function was
// supposed to end here. // supposed to end here.
try p.warnExpectedAfter(.expected_semi_or_lbrace); try p.warn(.expected_semi_or_lbrace);
return null_node; return null_node;
}, },
} }
@ -996,7 +1031,7 @@ const Parser = struct {
}; };
_ = p.eatToken(.keyword_else) orelse { _ = p.eatToken(.keyword_else) orelse {
if (else_required) { if (else_required) {
try p.warnExpectedAfter(.expected_semi_or_else); try p.warn(.expected_semi_or_else);
} }
return p.addNode(.{ return p.addNode(.{
.tag = .if_simple, .tag = .if_simple,
@ -1091,7 +1126,7 @@ const Parser = struct {
}; };
_ = p.eatToken(.keyword_else) orelse { _ = p.eatToken(.keyword_else) orelse {
if (else_required) { if (else_required) {
try p.warnExpectedAfter(.expected_semi_or_else); try p.warn(.expected_semi_or_else);
} }
return p.addNode(.{ return p.addNode(.{
.tag = .for_simple, .tag = .for_simple,
@ -1166,7 +1201,7 @@ const Parser = struct {
}; };
_ = p.eatToken(.keyword_else) orelse { _ = p.eatToken(.keyword_else) orelse {
if (else_required) { if (else_required) {
try p.warnExpectedAfter(.expected_semi_or_else); try p.warn(.expected_semi_or_else);
} }
if (cont_expr == 0) { if (cont_expr == 0) {
return p.addNode(.{ return p.addNode(.{
@ -2050,7 +2085,7 @@ const Parser = struct {
}, },
.colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_initializer), else => try p.warn(.expected_comma_after_initializer),
} }
if (p.eatToken(.r_brace)) |_| break; if (p.eatToken(.r_brace)) |_| break;
const next = try p.expectFieldInit(); const next = try p.expectFieldInit();
@ -2091,7 +2126,7 @@ const Parser = struct {
}, },
.colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_initializer), else => try p.warn(.expected_comma_after_initializer),
} }
} }
const comma = (p.token_tags[p.tok_i - 2] == .comma); const comma = (p.token_tags[p.tok_i - 2] == .comma);
@ -2170,7 +2205,7 @@ const Parser = struct {
}, },
.colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_arg), else => try p.warn(.expected_comma_after_arg),
} }
} }
const comma = (p.token_tags[p.tok_i - 2] == .comma); const comma = (p.token_tags[p.tok_i - 2] == .comma);
@ -2226,7 +2261,7 @@ const Parser = struct {
}, },
.colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_arg), else => try p.warn(.expected_comma_after_arg),
} }
} }
const comma = (p.token_tags[p.tok_i - 2] == .comma); const comma = (p.token_tags[p.tok_i - 2] == .comma);
@ -2467,7 +2502,7 @@ const Parser = struct {
}, },
.colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_initializer), else => try p.warn(.expected_comma_after_initializer),
} }
if (p.eatToken(.r_brace)) |_| break; if (p.eatToken(.r_brace)) |_| break;
const next = try p.expectFieldInit(); const next = try p.expectFieldInit();
@ -2519,7 +2554,7 @@ const Parser = struct {
}, },
.colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_initializer), else => try p.warn(.expected_comma_after_initializer),
} }
} }
const comma = (p.token_tags[p.tok_i - 2] == .comma); const comma = (p.token_tags[p.tok_i - 2] == .comma);
@ -2580,7 +2615,7 @@ const Parser = struct {
}, },
.colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_field), else => try p.warn(.expected_comma_after_field),
} }
} }
return p.addNode(.{ return p.addNode(.{
@ -2879,7 +2914,7 @@ const Parser = struct {
p.tok_i += 2; p.tok_i += 2;
return identifier; return identifier;
} }
return 0; return null_node;
} }
/// FieldInit <- DOT IDENTIFIER EQUAL Expr /// FieldInit <- DOT IDENTIFIER EQUAL Expr
@ -2896,9 +2931,12 @@ const Parser = struct {
} }
fn expectFieldInit(p: *Parser) !Node.Index { fn expectFieldInit(p: *Parser) !Node.Index {
_ = try p.expectToken(.period); if (p.token_tags[p.tok_i] != .period or
_ = try p.expectToken(.identifier); p.token_tags[p.tok_i + 1] != .identifier or
_ = try p.expectToken(.equal); p.token_tags[p.tok_i + 2] != .equal)
return p.fail(.expected_initializer);
p.tok_i += 3;
return p.expectExpr(); return p.expectExpr();
} }
@ -3413,7 +3451,7 @@ const Parser = struct {
// All possible delimiters. // All possible delimiters.
.colon, .r_paren, .r_brace, .r_bracket => break, .colon, .r_paren, .r_brace, .r_bracket => break,
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_switch_prong), else => try p.warn(.expected_comma_after_switch_prong),
} }
} }
return p.listToSpan(p.scratch.items[scratch_top..]); return p.listToSpan(p.scratch.items[scratch_top..]);
@ -3442,7 +3480,7 @@ const Parser = struct {
}, },
.colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren),
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_param), else => try p.warn(.expected_comma_after_param),
} }
} }
if (varargs == .nonfinal) { if (varargs == .nonfinal) {
@ -3486,7 +3524,7 @@ const Parser = struct {
break; break;
}, },
// Likely just a missing comma; give error but continue parsing. // Likely just a missing comma; give error but continue parsing.
else => try p.warnExpectedAfter(.expected_comma_after_arg), else => try p.warn(.expected_comma_after_arg),
} }
} }
const comma = (p.token_tags[p.tok_i - 2] == .comma); const comma = (p.token_tags[p.tok_i - 2] == .comma);
@ -3530,57 +3568,6 @@ const Parser = struct {
} }
} }
// string literal or multiline string literal
fn parseStringLiteral(p: *Parser) !Node.Index {
switch (p.token_tags[p.tok_i]) {
.string_literal => {
const main_token = p.nextToken();
return p.addNode(.{
.tag = .string_literal,
.main_token = main_token,
.data = .{
.lhs = undefined,
.rhs = undefined,
},
});
},
.multiline_string_literal_line => {
const first_line = p.nextToken();
while (p.token_tags[p.tok_i] == .multiline_string_literal_line) {
p.tok_i += 1;
}
return p.addNode(.{
.tag = .multiline_string_literal,
.main_token = first_line,
.data = .{
.lhs = first_line,
.rhs = p.tok_i - 1,
},
});
},
else => return null_node,
}
}
fn expectStringLiteral(p: *Parser) !Node.Index {
const node = try p.parseStringLiteral();
if (node == 0) {
return p.fail(.expected_string_literal);
}
return node;
}
fn expectIntegerLiteral(p: *Parser) !Node.Index {
return p.addNode(.{
.tag = .integer_literal,
.main_token = try p.expectToken(.integer_literal),
.data = .{
.lhs = undefined,
.rhs = undefined,
},
});
}
/// KEYWORD_if LPAREN Expr RPAREN PtrPayload? Body (KEYWORD_else Payload? Body)? /// KEYWORD_if LPAREN Expr RPAREN PtrPayload? Body (KEYWORD_else Payload? Body)?
fn parseIf(p: *Parser, bodyParseFn: fn (p: *Parser) Error!Node.Index) !Node.Index { fn parseIf(p: *Parser, bodyParseFn: fn (p: *Parser) Error!Node.Index) !Node.Index {
const if_token = p.eatToken(.keyword_if) orelse return null_node; const if_token = p.eatToken(.keyword_if) orelse return null_node;
@ -3649,25 +3636,14 @@ const Parser = struct {
} }
fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex { fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex {
const token = p.nextToken(); if (p.token_tags[p.tok_i] != tag) {
if (p.token_tags[token] != tag) {
p.tok_i -= 1; // Go back so that we can recover properly.
return p.failMsg(.{ return p.failMsg(.{
.tag = .expected_token, .tag = .expected_token,
.token = token, .token = p.tok_i,
.extra = .{ .expected_tag = tag }, .extra = .{ .expected_tag = tag },
}); });
} }
return token; return p.nextToken();
}
fn expectTokenRecoverable(p: *Parser, tag: Token.Tag) !?TokenIndex {
if (p.token_tags[p.tok_i] != tag) {
try p.warnExpected(tag);
return null;
} else {
return p.nextToken();
}
} }
fn expectSemicolon(p: *Parser, error_tag: AstError.Tag, recoverable: bool) Error!void { fn expectSemicolon(p: *Parser, error_tag: AstError.Tag, recoverable: bool) Error!void {
@ -3675,7 +3651,7 @@ const Parser = struct {
_ = p.nextToken(); _ = p.nextToken();
return; return;
} }
try p.warnExpectedAfter(error_tag); try p.warn(error_tag);
if (!recoverable) return error.ParseError; if (!recoverable) return error.ParseError;
} }

6
lib/std/zig/parser_test.zig
View file

@ -5057,7 +5057,9 @@ test "recovery: block statements" {
\\ inline; \\ inline;
\\} \\}
, &[_]Error{ , &[_]Error{
.invalid_token, .expected_expr,
.expected_semi_after_stmt,
.expected_statement,
.expected_inlinable, .expected_inlinable,
}); });
} }
@ -5076,7 +5078,7 @@ test "recovery: missing comma" {
, &[_]Error{ , &[_]Error{
.expected_comma_after_switch_prong, .expected_comma_after_switch_prong,
.expected_comma_after_switch_prong, .expected_comma_after_switch_prong,
.invalid_token, .expected_expr,
}); });
} }

13
lib/std/zig/tokenizer.zig
View file

@ -322,7 +322,18 @@ pub const Token = struct {
} }
pub fn symbol(tag: Tag) []const u8 { pub fn symbol(tag: Tag) []const u8 {
return tag.lexeme() orelse @tagName(tag); return tag.lexeme() orelse switch (tag) {
.invalid => "invalid bytes",
.identifier => "an identifier",
.string_literal, .multiline_string_literal_line => "a string literal",
.char_literal => "a character literal",
.eof => "EOF",
.builtin => "a builtin function",
.integer_literal => "an integer literal",
.float_literal => "a floating point literal",
.doc_comment, .container_doc_comment => "a document comment",
else => unreachable,
};
} }
}; };
}; };

8
src/Module.zig
View file

@ -2995,7 +2995,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
const token_starts = file.tree.tokens.items(.start); const token_starts = file.tree.tokens.items(.start);
const token_tags = file.tree.tokens.items(.tag); const token_tags = file.tree.tokens.items(.tag);
const extra_offset = file.tree.errorOffset(parse_err.tag, parse_err.token); const extra_offset = file.tree.errorOffset(parse_err);
try file.tree.renderError(parse_err, msg.writer()); try file.tree.renderError(parse_err, msg.writer());
const err_msg = try gpa.create(ErrorMsg); const err_msg = try gpa.create(ErrorMsg);
err_msg.* = .{ err_msg.* = .{
@ -3006,9 +3006,9 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
}, },
.msg = msg.toOwnedSlice(), .msg = msg.toOwnedSlice(),
}; };
if (token_tags[parse_err.token] == .invalid) { if (token_tags[parse_err.token + @boolToInt(parse_err.token_is_prev)] == .invalid) {
const bad_off = @intCast(u32, file.tree.tokenSlice(parse_err.token).len); const bad_off = @intCast(u32, file.tree.tokenSlice(parse_err.token + @boolToInt(parse_err.token_is_prev)).len);
const byte_abs = token_starts[parse_err.token] + bad_off; const byte_abs = token_starts[parse_err.token + @boolToInt(parse_err.token_is_prev)] + bad_off;
try mod.errNoteNonLazy(.{ try mod.errNoteNonLazy(.{
.file_scope = file, .file_scope = file,
.parent_decl_node = 0, .parent_decl_node = 0,

8
src/main.zig
View file

@ -4063,9 +4063,9 @@ fn printErrMsgToStdErr(
var notes_buffer: [1]Compilation.AllErrors.Message = undefined; var notes_buffer: [1]Compilation.AllErrors.Message = undefined;
var notes_len: usize = 0; var notes_len: usize = 0;
if (token_tags[parse_error.token] == .invalid) { if (token_tags[parse_error.token + @boolToInt(parse_error.token_is_prev)] == .invalid) {
const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token).len); const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token + @boolToInt(parse_error.token_is_prev)).len);
const byte_offset = @intCast(u32, start_loc.line_start) + bad_off; const byte_offset = @intCast(u32, start_loc.line_start) + @intCast(u32, start_loc.column) + bad_off;
notes_buffer[notes_len] = .{ notes_buffer[notes_len] = .{
.src = .{ .src = .{
.src_path = path, .src_path = path,
@ -4081,7 +4081,7 @@ fn printErrMsgToStdErr(
notes_len += 1; notes_len += 1;
} }
const extra_offset = tree.errorOffset(parse_error.tag, parse_error.token); const extra_offset = tree.errorOffset(parse_error);
const message: Compilation.AllErrors.Message = .{ const message: Compilation.AllErrors.Message = .{
.src = .{ .src = .{
.src_path = path, .src_path = path,

48
test/compile_errors.zig
View file

@ -1540,7 +1540,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ std.debug.assert(bad_float < 1.0); \\ std.debug.assert(bad_float < 1.0);
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:5:29: error: invalid token: '.'", "tmp.zig:5:29: error: expected expression, found '.'",
}); });
ctx.objErrStage1("invalid exponent in float literal - 1", ctx.objErrStage1("invalid exponent in float literal - 1",
@ -1549,7 +1549,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: 'a'", "tmp.zig:2:28: note: invalid byte: 'a'",
}); });
@ -1559,7 +1559,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:29: note: invalid byte: 'F'", "tmp.zig:2:29: note: invalid byte: 'F'",
}); });
@ -1569,7 +1569,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:23: note: invalid byte: '_'", "tmp.zig:2:23: note: invalid byte: '_'",
}); });
@ -1579,7 +1579,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:23: note: invalid byte: '.'", "tmp.zig:2:23: note: invalid byte: '.'",
}); });
@ -1589,7 +1589,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:25: note: invalid byte: ';'", "tmp.zig:2:25: note: invalid byte: ';'",
}); });
@ -1599,7 +1599,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:25: note: invalid byte: '_'", "tmp.zig:2:25: note: invalid byte: '_'",
}); });
@ -1609,7 +1609,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:26: note: invalid byte: '_'", "tmp.zig:2:26: note: invalid byte: '_'",
}); });
@ -1619,7 +1619,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:26: note: invalid byte: '_'", "tmp.zig:2:26: note: invalid byte: '_'",
}); });
@ -1629,7 +1629,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: ';'", "tmp.zig:2:28: note: invalid byte: ';'",
}); });
@ -1639,7 +1639,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:23: note: invalid byte: '_'", "tmp.zig:2:23: note: invalid byte: '_'",
}); });
@ -1649,7 +1649,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:25: note: invalid byte: '_'", "tmp.zig:2:25: note: invalid byte: '_'",
}); });
@ -1659,7 +1659,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: '_'", "tmp.zig:2:28: note: invalid byte: '_'",
}); });
@ -1669,7 +1669,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:23: note: invalid byte: 'x'", "tmp.zig:2:23: note: invalid byte: 'x'",
}); });
@ -1679,7 +1679,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:23: note: invalid byte: '_'", "tmp.zig:2:23: note: invalid byte: '_'",
}); });
@ -1689,7 +1689,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:27: note: invalid byte: 'p'", "tmp.zig:2:27: note: invalid byte: 'p'",
}); });
@ -1699,7 +1699,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:26: note: invalid byte: ';'", "tmp.zig:2:26: note: invalid byte: ';'",
}); });
@ -1709,7 +1709,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: ';'", "tmp.zig:2:28: note: invalid byte: ';'",
}); });
@ -1719,7 +1719,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: ';'", "tmp.zig:2:28: note: invalid byte: ';'",
}); });
@ -1729,7 +1729,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = bad; \\ _ = bad;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:21: error: expected expression, found 'invalid'", "tmp.zig:2:21: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:28: note: invalid byte: ';'", "tmp.zig:2:28: note: invalid byte: ';'",
}); });
@ -2171,7 +2171,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = x; \\ _ = x;
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:3:6: error: expected ',' after field", "tmp.zig:3:7: error: expected ',' after field",
}); });
ctx.objErrStage1("bad alignment type", ctx.objErrStage1("bad alignment type",
@ -5733,7 +5733,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\const foo = "a \\const foo = "a
\\b"; \\b";
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:1:13: error: expected expression, found 'invalid'", "tmp.zig:1:13: error: expected expression, found 'invalid bytes'",
"tmp.zig:1:15: note: invalid byte: '\\n'", "tmp.zig:1:15: note: invalid byte: '\\n'",
}); });
@ -7638,7 +7638,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ const a = '\U1234'; \\ const a = '\U1234';
\\} \\}
, &[_][]const u8{ , &[_][]const u8{
"tmp.zig:2:15: error: expected expression, found 'invalid'", "tmp.zig:2:15: error: expected expression, found 'invalid bytes'",
"tmp.zig:2:18: note: invalid byte: '1'", "tmp.zig:2:18: note: invalid byte: '1'",
}); });
@ -7654,7 +7654,7 @@ pub fn addCases(ctx: *TestContext) !void {
"fn foo() bool {\r\n" ++ "fn foo() bool {\r\n" ++
" return true;\r\n" ++ " return true;\r\n" ++
"}\r\n", &[_][]const u8{ "}\r\n", &[_][]const u8{
"tmp.zig:1:1: error: expected test, comptime, var decl, or container field, found 'invalid'", "tmp.zig:1:1: error: expected test, comptime, var decl, or container field, found 'invalid bytes'",
"tmp.zig:1:1: note: invalid byte: '\\xff'", "tmp.zig:1:1: note: invalid byte: '\\xff'",
}); });

2
test/stage2/cbe.zig
View file

@ -693,7 +693,7 @@ pub fn addCases(ctx: *TestContext) !void {
\\ _ = E1.a; \\ _ = E1.a;
\\} \\}
, &.{ , &.{
":3:6: error: expected ',' after field", ":3:7: error: expected ',' after field",
}); });
// Redundant non-exhaustive enum mark. // Redundant non-exhaustive enum mark.