translate-c: fix c tokenizer giving invalid tokens

This commit is contained in:
Veikka Tuominen 2021-03-03 11:58:09 +02:00 committed by Andrew Kelley
parent 7fbe9e7d60
commit 904f774563
3 changed files with 33 additions and 3 deletions

View file

@ -401,7 +401,9 @@ pub const Tokenizer = struct {
Zero, Zero,
IntegerLiteralOct, IntegerLiteralOct,
IntegerLiteralBinary, IntegerLiteralBinary,
IntegerLiteralBinaryFirst,
IntegerLiteralHex, IntegerLiteralHex,
IntegerLiteralHexFirst,
IntegerLiteral, IntegerLiteral,
IntegerSuffix, IntegerSuffix,
IntegerSuffixU, IntegerSuffixU,
@ -1046,10 +1048,10 @@ pub const Tokenizer = struct {
state = .IntegerLiteralOct; state = .IntegerLiteralOct;
}, },
'b', 'B' => { 'b', 'B' => {
state = .IntegerLiteralBinary; state = .IntegerLiteralBinaryFirst;
}, },
'x', 'X' => { 'x', 'X' => {
state = .IntegerLiteralHex; state = .IntegerLiteralHexFirst;
}, },
'.' => { '.' => {
state = .FloatFraction; state = .FloatFraction;
@ -1066,6 +1068,13 @@ pub const Tokenizer = struct {
self.index -= 1; self.index -= 1;
}, },
}, },
.IntegerLiteralBinaryFirst => switch (c) {
'0'...'7' => state = .IntegerLiteralBinary,
else => {
result.id = .Invalid;
break;
},
},
.IntegerLiteralBinary => switch (c) { .IntegerLiteralBinary => switch (c) {
'0', '1' => {}, '0', '1' => {},
else => { else => {
@ -1073,6 +1082,19 @@ pub const Tokenizer = struct {
self.index -= 1; self.index -= 1;
}, },
}, },
.IntegerLiteralHexFirst => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => state = .IntegerLiteralHex,
'.' => {
state = .FloatFractionHex;
},
'p', 'P' => {
state = .FloatExponent;
},
else => {
result.id = .Invalid;
break;
},
},
.IntegerLiteralHex => switch (c) { .IntegerLiteralHex => switch (c) {
'0'...'9', 'a'...'f', 'A'...'F' => {}, '0'...'9', 'a'...'f', 'A'...'F' => {},
'.' => { '.' => {
@ -1238,6 +1260,8 @@ pub const Tokenizer = struct {
.MultiLineCommentAsterisk, .MultiLineCommentAsterisk,
.FloatExponent, .FloatExponent,
.MacroString, .MacroString,
.IntegerLiteralBinaryFirst,
.IntegerLiteralHexFirst,
=> result.id = .Invalid, => result.id = .Invalid,
.FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none }, .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none },
@ -1523,6 +1547,7 @@ test "num suffixes" {
\\ 1.0f 1.0L 1.0 .0 1. \\ 1.0f 1.0L 1.0 .0 1.
\\ 0l 0lu 0ll 0llu 0 \\ 0l 0lu 0ll 0llu 0
\\ 1u 1ul 1ull 1 \\ 1u 1ul 1ull 1
\\ 0x 0b
\\ \\
, &[_]Token.Id{ , &[_]Token.Id{
.{ .FloatLiteral = .f }, .{ .FloatLiteral = .f },
@ -1542,6 +1567,9 @@ test "num suffixes" {
.{ .IntegerLiteral = .llu }, .{ .IntegerLiteral = .llu },
.{ .IntegerLiteral = .none }, .{ .IntegerLiteral = .none },
.Nl, .Nl,
.Invalid,
.Invalid,
.Nl,
}); });
} }

View file

@ -136,7 +136,7 @@ pub fn MultiArrayList(comptime S: type) type {
const slices = self.slice(); const slices = self.slice();
var result: S = undefined; var result: S = undefined;
inline for (fields) |field_info, i| { inline for (fields) |field_info, i| {
@field(elem, field_info.name) = slices.items(@intToEnum(Field, i))[index]; @field(result, field_info.name) = slices.items(@intToEnum(Field, i))[index];
} }
return result; return result;
} }

View file

@ -27,6 +27,8 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void {
\\#define FOO = \\#define FOO =
\\#define PtrToPtr64(p) ((void *POINTER_64) p) \\#define PtrToPtr64(p) ((void *POINTER_64) p)
\\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s)) \\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s))
\\#define bar = 0x
\\#define baz = 0b
\\int main(void) {} \\int main(void) {}
, ""); , "");