mirror of
https://codeberg.org/ziglang/zig.git
synced 2025-12-06 13:54:21 +00:00
Merge pull request #25034 from ziglang/lzma
std.compress: update lzma, lzma2, and xz to new I/O API
This commit is contained in:
Andrew Kelley
GitHub
commit
50edad37ba
13 changed files with 1470 additions and 1612 deletions
|
|
@ -2,89 +2,759 @@ const std = @import("../std.zig");
|
|||
const math = std.math;
|
||||
const mem = std.mem;
|
||||
const Allocator = std.mem.Allocator;
|
||||
const assert = std.debug.assert;
|
||||
const ArrayList = std.ArrayList;
|
||||
const Writer = std.Io.Writer;
|
||||
const Reader = std.Io.Reader;
|
||||
|
||||
pub const decode = @import("lzma/decode.zig");
|
||||
pub const RangeDecoder = struct {
|
||||
range: u32,
|
||||
code: u32,
|
||||
|
||||
pub fn decompress(
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
) !Decompress(@TypeOf(reader)) {
|
||||
return decompressWithOptions(allocator, reader, .{});
|
||||
}
|
||||
pub fn init(reader: *Reader) !RangeDecoder {
|
||||
var counter: u64 = 0;
|
||||
return initCounting(reader, &counter);
|
||||
}
|
||||
|
||||
pub fn decompressWithOptions(
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
options: decode.Options,
|
||||
) !Decompress(@TypeOf(reader)) {
|
||||
const params = try decode.Params.readHeader(reader, options);
|
||||
return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);
|
||||
}
|
||||
|
||||
pub fn Decompress(comptime ReaderType: type) type {
|
||||
return struct {
|
||||
const Self = @This();
|
||||
|
||||
pub const Error =
|
||||
ReaderType.Error ||
|
||||
Allocator.Error ||
|
||||
error{ CorruptInput, EndOfStream, Overflow };
|
||||
|
||||
pub const Reader = std.io.GenericReader(*Self, Error, read);
|
||||
|
||||
allocator: Allocator,
|
||||
in_reader: ReaderType,
|
||||
to_read: std.ArrayListUnmanaged(u8),
|
||||
|
||||
buffer: decode.lzbuffer.LzCircularBuffer,
|
||||
decoder: decode.rangecoder.RangeDecoder,
|
||||
state: decode.DecoderState,
|
||||
|
||||
pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {
|
||||
return Self{
|
||||
.allocator = allocator,
|
||||
.in_reader = source,
|
||||
.to_read = .{},
|
||||
|
||||
.buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
|
||||
.decoder = try decode.rangecoder.RangeDecoder.init(source),
|
||||
.state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),
|
||||
pub fn initCounting(reader: *Reader, n_read: *u64) !RangeDecoder {
|
||||
const reserved = try reader.takeByte();
|
||||
n_read.* += 1;
|
||||
if (reserved != 0) return error.InvalidRangeCode;
|
||||
const code = try reader.takeInt(u32, .big);
|
||||
n_read.* += 4;
|
||||
return .{
|
||||
.range = 0xFFFF_FFFF,
|
||||
.code = code,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn reader(self: *Self) Reader {
|
||||
return .{ .context = self };
|
||||
pub fn isFinished(self: RangeDecoder) bool {
|
||||
return self.code == 0;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.to_read.deinit(self.allocator);
|
||||
self.buffer.deinit(self.allocator);
|
||||
self.state.deinit(self.allocator);
|
||||
fn normalize(self: *RangeDecoder, reader: *Reader, n_read: *u64) !void {
|
||||
if (self.range < 0x0100_0000) {
|
||||
self.range <<= 8;
|
||||
self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
|
||||
n_read.* += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn getBit(self: *RangeDecoder, reader: *Reader, n_read: *u64) !bool {
|
||||
self.range >>= 1;
|
||||
|
||||
const bit = self.code >= self.range;
|
||||
if (bit) self.code -= self.range;
|
||||
|
||||
try self.normalize(reader, n_read);
|
||||
return bit;
|
||||
}
|
||||
|
||||
pub fn get(self: *RangeDecoder, reader: *Reader, count: usize, n_read: *u64) !u32 {
|
||||
var result: u32 = 0;
|
||||
for (0..count) |_| {
|
||||
result = (result << 1) ^ @intFromBool(try self.getBit(reader, n_read));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16, n_read: *u64) !bool {
|
||||
const bound = (self.range >> 11) * prob.*;
|
||||
|
||||
if (self.code < bound) {
|
||||
prob.* += (0x800 - prob.*) >> 5;
|
||||
self.range = bound;
|
||||
|
||||
try self.normalize(reader, n_read);
|
||||
return false;
|
||||
} else {
|
||||
prob.* -= prob.* >> 5;
|
||||
self.code -= bound;
|
||||
self.range -= bound;
|
||||
|
||||
try self.normalize(reader, n_read);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
fn parseBitTree(
|
||||
self: *RangeDecoder,
|
||||
reader: *Reader,
|
||||
num_bits: u5,
|
||||
probs: []u16,
|
||||
n_read: *u64,
|
||||
) !u32 {
|
||||
var tmp: u32 = 1;
|
||||
var i: @TypeOf(num_bits) = 0;
|
||||
while (i < num_bits) : (i += 1) {
|
||||
const bit = try self.decodeBit(reader, &probs[tmp], n_read);
|
||||
tmp = (tmp << 1) ^ @intFromBool(bit);
|
||||
}
|
||||
return tmp - (@as(u32, 1) << num_bits);
|
||||
}
|
||||
|
||||
pub fn parseReverseBitTree(
|
||||
self: *RangeDecoder,
|
||||
reader: *Reader,
|
||||
num_bits: u5,
|
||||
probs: []u16,
|
||||
offset: usize,
|
||||
n_read: *u64,
|
||||
) !u32 {
|
||||
var result: u32 = 0;
|
||||
var tmp: usize = 1;
|
||||
var i: @TypeOf(num_bits) = 0;
|
||||
while (i < num_bits) : (i += 1) {
|
||||
const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], n_read));
|
||||
tmp = (tmp << 1) ^ bit;
|
||||
result ^= @as(u32, bit) << i;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
pub const Decode = struct {
|
||||
properties: Properties,
|
||||
literal_probs: Vec2d,
|
||||
pos_slot_decoder: [4]BitTree(6),
|
||||
align_decoder: BitTree(4),
|
||||
pos_decoders: [115]u16,
|
||||
is_match: [192]u16,
|
||||
is_rep: [12]u16,
|
||||
is_rep_g0: [12]u16,
|
||||
is_rep_g1: [12]u16,
|
||||
is_rep_g2: [12]u16,
|
||||
is_rep_0long: [192]u16,
|
||||
state: usize,
|
||||
rep: [4]usize,
|
||||
len_decoder: LenDecoder,
|
||||
rep_len_decoder: LenDecoder,
|
||||
|
||||
pub fn init(gpa: Allocator, properties: Properties) !Decode {
|
||||
return .{
|
||||
.properties = properties,
|
||||
.literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (properties.lc + properties.lp), 0x300),
|
||||
.pos_slot_decoder = @splat(.{}),
|
||||
.align_decoder = .{},
|
||||
.pos_decoders = @splat(0x400),
|
||||
.is_match = @splat(0x400),
|
||||
.is_rep = @splat(0x400),
|
||||
.is_rep_g0 = @splat(0x400),
|
||||
.is_rep_g1 = @splat(0x400),
|
||||
.is_rep_g2 = @splat(0x400),
|
||||
.is_rep_0long = @splat(0x400),
|
||||
.state = 0,
|
||||
.rep = @splat(0),
|
||||
.len_decoder = .{},
|
||||
.rep_len_decoder = .{},
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Decode, gpa: Allocator) void {
|
||||
self.literal_probs.deinit(gpa);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn read(self: *Self, output: []u8) Error!usize {
|
||||
const writer = self.to_read.writer(self.allocator);
|
||||
while (self.to_read.items.len < output.len) {
|
||||
switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
|
||||
.continue_ => {},
|
||||
.finished => {
|
||||
try self.buffer.finish(writer);
|
||||
pub fn resetState(self: *Decode, gpa: Allocator, new_props: Properties) !void {
|
||||
new_props.validate();
|
||||
if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
|
||||
self.literal_probs.fill(0x400);
|
||||
} else {
|
||||
self.literal_probs.deinit(gpa);
|
||||
self.literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
|
||||
}
|
||||
|
||||
self.properties = new_props;
|
||||
for (&self.pos_slot_decoder) |*t| t.reset();
|
||||
self.align_decoder.reset();
|
||||
self.pos_decoders = @splat(0x400);
|
||||
self.is_match = @splat(0x400);
|
||||
self.is_rep = @splat(0x400);
|
||||
self.is_rep_g0 = @splat(0x400);
|
||||
self.is_rep_g1 = @splat(0x400);
|
||||
self.is_rep_g2 = @splat(0x400);
|
||||
self.is_rep_0long = @splat(0x400);
|
||||
self.state = 0;
|
||||
self.rep = @splat(0);
|
||||
self.len_decoder.reset();
|
||||
self.rep_len_decoder.reset();
|
||||
}
|
||||
|
||||
pub fn process(
|
||||
self: *Decode,
|
||||
reader: *Reader,
|
||||
allocating: *Writer.Allocating,
|
||||
/// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
n_read: *u64,
|
||||
) !ProcessingStatus {
|
||||
const gpa = allocating.allocator;
|
||||
const writer = &allocating.writer;
|
||||
const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
|
||||
|
||||
if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], n_read)) {
|
||||
const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, n_read);
|
||||
|
||||
try buffer.appendLiteral(gpa, byte, writer);
|
||||
|
||||
self.state = if (self.state < 4)
|
||||
0
|
||||
else if (self.state < 10)
|
||||
self.state - 3
|
||||
else
|
||||
self.state - 6;
|
||||
return .more;
|
||||
}
|
||||
|
||||
var len: usize = undefined;
|
||||
if (try decoder.decodeBit(reader, &self.is_rep[self.state], n_read)) {
|
||||
if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], n_read)) {
|
||||
if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state], n_read)) {
|
||||
self.state = if (self.state < 7) 9 else 11;
|
||||
const dist = self.rep[0] + 1;
|
||||
try buffer.appendLz(gpa, 1, dist, writer);
|
||||
return .more;
|
||||
}
|
||||
} else {
|
||||
const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], n_read))
|
||||
1
|
||||
else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], n_read))
|
||||
2
|
||||
else
|
||||
3;
|
||||
const dist = self.rep[idx];
|
||||
var i = idx;
|
||||
while (i > 0) : (i -= 1) {
|
||||
self.rep[i] = self.rep[i - 1];
|
||||
}
|
||||
self.rep[0] = dist;
|
||||
}
|
||||
|
||||
len = try self.rep_len_decoder.decode(reader, decoder, pos_state, n_read);
|
||||
|
||||
self.state = if (self.state < 7) 8 else 11;
|
||||
} else {
|
||||
self.rep[3] = self.rep[2];
|
||||
self.rep[2] = self.rep[1];
|
||||
self.rep[1] = self.rep[0];
|
||||
|
||||
len = try self.len_decoder.decode(reader, decoder, pos_state, n_read);
|
||||
|
||||
self.state = if (self.state < 7) 7 else 10;
|
||||
|
||||
const rep_0 = try self.decodeDistance(reader, decoder, len, n_read);
|
||||
|
||||
self.rep[0] = rep_0;
|
||||
if (self.rep[0] == 0xFFFF_FFFF) {
|
||||
if (decoder.isFinished()) {
|
||||
return .finished;
|
||||
}
|
||||
return error.CorruptInput;
|
||||
}
|
||||
}
|
||||
|
||||
len += 2;
|
||||
|
||||
const dist = self.rep[0] + 1;
|
||||
try buffer.appendLz(gpa, len, dist, writer);
|
||||
|
||||
return .more;
|
||||
}
|
||||
|
||||
fn decodeLiteral(
|
||||
self: *Decode,
|
||||
reader: *Reader,
|
||||
/// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
n_read: *u64,
|
||||
) !u8 {
|
||||
const def_prev_byte = 0;
|
||||
const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
|
||||
|
||||
var result: usize = 1;
|
||||
const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
|
||||
(prev_byte >> (8 - self.properties.lc));
|
||||
const probs = try self.literal_probs.get(lit_state);
|
||||
|
||||
if (self.state >= 7) {
|
||||
var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
|
||||
|
||||
while (result < 0x100) {
|
||||
const match_bit = (match_byte >> 7) & 1;
|
||||
match_byte <<= 1;
|
||||
const bit = @intFromBool(try decoder.decodeBit(
|
||||
reader,
|
||||
&probs[((@as(usize, 1) + match_bit) << 8) + result],
|
||||
n_read,
|
||||
));
|
||||
result = (result << 1) ^ bit;
|
||||
if (match_bit != bit) {
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
const input = self.to_read.items;
|
||||
const n = @min(input.len, output.len);
|
||||
@memcpy(output[0..n], input[0..n]);
|
||||
std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
|
||||
self.to_read.shrinkRetainingCapacity(input.len - n);
|
||||
return n;
|
||||
}
|
||||
|
||||
while (result < 0x100) {
|
||||
result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], n_read));
|
||||
}
|
||||
|
||||
return @truncate(result - 0x100);
|
||||
}
|
||||
|
||||
fn decodeDistance(
|
||||
self: *Decode,
|
||||
reader: *Reader,
|
||||
decoder: *RangeDecoder,
|
||||
length: usize,
|
||||
n_read: *u64,
|
||||
) !usize {
|
||||
const len_state = if (length > 3) 3 else length;
|
||||
|
||||
const pos_slot: usize = try self.pos_slot_decoder[len_state].parse(reader, decoder, n_read);
|
||||
if (pos_slot < 4) return pos_slot;
|
||||
|
||||
const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
|
||||
var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
|
||||
|
||||
if (pos_slot < 14) {
|
||||
result += try decoder.parseReverseBitTree(
|
||||
reader,
|
||||
num_direct_bits,
|
||||
&self.pos_decoders,
|
||||
result - pos_slot,
|
||||
n_read,
|
||||
);
|
||||
} else {
|
||||
result += @as(usize, try decoder.get(reader, num_direct_bits - 4, n_read)) << 4;
|
||||
result += try self.align_decoder.parseReverse(reader, decoder, n_read);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// A circular buffer for LZ sequences
|
||||
pub const CircularBuffer = struct {
|
||||
/// Circular buffer
|
||||
buf: ArrayList(u8),
|
||||
/// Length of the buffer
|
||||
dict_size: usize,
|
||||
/// Buffer memory limit
|
||||
mem_limit: usize,
|
||||
/// Current position
|
||||
cursor: usize,
|
||||
/// Total number of bytes sent through the buffer
|
||||
len: usize,
|
||||
|
||||
pub fn init(dict_size: usize, mem_limit: usize) CircularBuffer {
|
||||
return .{
|
||||
.buf = .{},
|
||||
.dict_size = dict_size,
|
||||
.mem_limit = mem_limit,
|
||||
.cursor = 0,
|
||||
.len = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn get(self: CircularBuffer, index: usize) u8 {
|
||||
return if (0 <= index and index < self.buf.items.len) self.buf.items[index] else 0;
|
||||
}
|
||||
|
||||
pub fn set(self: *CircularBuffer, gpa: Allocator, index: usize, value: u8) !void {
|
||||
if (index >= self.mem_limit) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
try self.buf.ensureTotalCapacity(gpa, index + 1);
|
||||
while (self.buf.items.len < index) {
|
||||
self.buf.appendAssumeCapacity(0);
|
||||
}
|
||||
self.buf.appendAssumeCapacity(value);
|
||||
}
|
||||
|
||||
/// Retrieve the last byte or return a default
|
||||
pub fn lastOr(self: CircularBuffer, lit: u8) u8 {
|
||||
return if (self.len == 0)
|
||||
lit
|
||||
else
|
||||
self.get((self.dict_size + self.cursor - 1) % self.dict_size);
|
||||
}
|
||||
|
||||
/// Retrieve the n-th last byte
|
||||
pub fn lastN(self: CircularBuffer, dist: usize) !u8 {
|
||||
if (dist > self.dict_size or dist > self.len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
|
||||
return self.get(offset);
|
||||
}
|
||||
|
||||
/// Append a literal
|
||||
pub fn appendLiteral(
|
||||
self: *CircularBuffer,
|
||||
gpa: Allocator,
|
||||
lit: u8,
|
||||
writer: *Writer,
|
||||
) !void {
|
||||
try self.set(gpa, self.cursor, lit);
|
||||
self.cursor += 1;
|
||||
self.len += 1;
|
||||
|
||||
// Flush the circular buffer to the output
|
||||
if (self.cursor == self.dict_size) {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.cursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch an LZ sequence (length, distance) from inside the buffer
|
||||
pub fn appendLz(
|
||||
self: *CircularBuffer,
|
||||
gpa: Allocator,
|
||||
len: usize,
|
||||
dist: usize,
|
||||
writer: *Writer,
|
||||
) !void {
|
||||
if (dist > self.dict_size or dist > self.len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
|
||||
var i: usize = 0;
|
||||
while (i < len) : (i += 1) {
|
||||
const x = self.get(offset);
|
||||
try self.appendLiteral(gpa, x, writer);
|
||||
offset += 1;
|
||||
if (offset == self.dict_size) {
|
||||
offset = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(self: *CircularBuffer, writer: *Writer) !void {
|
||||
if (self.cursor > 0) {
|
||||
try writer.writeAll(self.buf.items[0..self.cursor]);
|
||||
self.cursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deinit(self: *CircularBuffer, gpa: Allocator) void {
|
||||
self.buf.deinit(gpa);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn BitTree(comptime num_bits: usize) type {
|
||||
return struct {
|
||||
probs: [1 << num_bits]u16 = @splat(0x400),
|
||||
|
||||
pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder, n_read: *u64) !u32 {
|
||||
return decoder.parseBitTree(reader, num_bits, &self.probs, n_read);
|
||||
}
|
||||
|
||||
pub fn parseReverse(
|
||||
self: *@This(),
|
||||
reader: *Reader,
|
||||
decoder: *RangeDecoder,
|
||||
n_read: *u64,
|
||||
) !u32 {
|
||||
return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, n_read);
|
||||
}
|
||||
|
||||
pub fn reset(self: *@This()) void {
|
||||
@memset(&self.probs, 0x400);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub const LenDecoder = struct {
|
||||
choice: u16 = 0x400,
|
||||
choice2: u16 = 0x400,
|
||||
low_coder: [16]BitTree(3) = @splat(.{}),
|
||||
mid_coder: [16]BitTree(3) = @splat(.{}),
|
||||
high_coder: BitTree(8) = .{},
|
||||
|
||||
pub fn decode(
|
||||
self: *LenDecoder,
|
||||
reader: *Reader,
|
||||
decoder: *RangeDecoder,
|
||||
pos_state: usize,
|
||||
n_read: *u64,
|
||||
) !usize {
|
||||
if (!try decoder.decodeBit(reader, &self.choice, n_read)) {
|
||||
return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, n_read));
|
||||
} else if (!try decoder.decodeBit(reader, &self.choice2, n_read)) {
|
||||
return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, n_read)) + 8;
|
||||
} else {
|
||||
return @as(usize, try self.high_coder.parse(reader, decoder, n_read)) + 16;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(self: *LenDecoder) void {
|
||||
self.choice = 0x400;
|
||||
self.choice2 = 0x400;
|
||||
for (&self.low_coder) |*t| t.reset();
|
||||
for (&self.mid_coder) |*t| t.reset();
|
||||
self.high_coder.reset();
|
||||
}
|
||||
};
|
||||
|
||||
pub const Vec2d = struct {
|
||||
data: []u16,
|
||||
cols: usize,
|
||||
|
||||
pub fn init(gpa: Allocator, value: u16, w: usize, h: usize) !Vec2d {
|
||||
const len = try math.mul(usize, w, h);
|
||||
const data = try gpa.alloc(u16, len);
|
||||
@memset(data, value);
|
||||
return .{
|
||||
.data = data,
|
||||
.cols = h,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(v: *Vec2d, gpa: Allocator) void {
|
||||
gpa.free(v.data);
|
||||
v.* = undefined;
|
||||
}
|
||||
|
||||
pub fn fill(v: *Vec2d, value: u16) void {
|
||||
@memset(v.data, value);
|
||||
}
|
||||
|
||||
fn get(v: Vec2d, row: usize) ![]u16 {
|
||||
const start_row = try math.mul(usize, row, v.cols);
|
||||
const end_row = try math.add(usize, start_row, v.cols);
|
||||
return v.data[start_row..end_row];
|
||||
}
|
||||
};
|
||||
|
||||
pub const Options = struct {
|
||||
unpacked_size: UnpackedSize = .read_from_header,
|
||||
mem_limit: ?usize = null,
|
||||
allow_incomplete: bool = false,
|
||||
};
|
||||
|
||||
pub const UnpackedSize = union(enum) {
|
||||
read_from_header,
|
||||
read_header_but_use_provided: ?u64,
|
||||
use_provided: ?u64,
|
||||
};
|
||||
|
||||
const ProcessingStatus = enum {
|
||||
more,
|
||||
finished,
|
||||
};
|
||||
|
||||
pub const Properties = struct {
|
||||
lc: u4,
|
||||
lp: u3,
|
||||
pb: u3,
|
||||
|
||||
fn validate(self: Properties) void {
|
||||
assert(self.lc <= 8);
|
||||
assert(self.lp <= 4);
|
||||
assert(self.pb <= 4);
|
||||
}
|
||||
};
|
||||
|
||||
pub const Params = struct {
|
||||
properties: Properties,
|
||||
dict_size: u32,
|
||||
unpacked_size: ?u64,
|
||||
|
||||
pub fn readHeader(reader: *Reader, options: Options) !Params {
|
||||
var props = try reader.takeByte();
|
||||
if (props >= 225) return error.CorruptInput;
|
||||
|
||||
const lc: u4 = @intCast(props % 9);
|
||||
props /= 9;
|
||||
const lp: u3 = @intCast(props % 5);
|
||||
props /= 5;
|
||||
const pb: u3 = @intCast(props);
|
||||
|
||||
const dict_size_provided = try reader.takeInt(u32, .little);
|
||||
const dict_size = @max(0x1000, dict_size_provided);
|
||||
|
||||
const unpacked_size = switch (options.unpacked_size) {
|
||||
.read_from_header => blk: {
|
||||
const unpacked_size_provided = try reader.takeInt(u64, .little);
|
||||
const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
|
||||
break :blk if (marker_mandatory) null else unpacked_size_provided;
|
||||
},
|
||||
.read_header_but_use_provided => |x| blk: {
|
||||
_ = try reader.takeInt(u64, .little);
|
||||
break :blk x;
|
||||
},
|
||||
.use_provided => |x| x,
|
||||
};
|
||||
|
||||
return .{
|
||||
.properties = .{ .lc = lc, .lp = lp, .pb = pb },
|
||||
.dict_size = dict_size,
|
||||
.unpacked_size = unpacked_size,
|
||||
};
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
pub const Decompress = struct {
|
||||
gpa: Allocator,
|
||||
input: *Reader,
|
||||
reader: Reader,
|
||||
buffer: Decode.CircularBuffer,
|
||||
range_decoder: RangeDecoder,
|
||||
decode: Decode,
|
||||
err: ?Error,
|
||||
unpacked_size: ?u64,
|
||||
|
||||
pub const Error = error{
|
||||
OutOfMemory,
|
||||
ReadFailed,
|
||||
CorruptInput,
|
||||
DecompressedSizeMismatch,
|
||||
EndOfStream,
|
||||
Overflow,
|
||||
};
|
||||
|
||||
/// Takes ownership of `buffer` which may be resized with `gpa`.
|
||||
///
|
||||
/// LZMA was explicitly designed to take advantage of large heap memory
|
||||
/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
|
||||
/// this API dynamically allocates the dictionary as-needed.
|
||||
pub fn initParams(
|
||||
input: *Reader,
|
||||
gpa: Allocator,
|
||||
buffer: []u8,
|
||||
params: Decode.Params,
|
||||
mem_limit: usize,
|
||||
) !Decompress {
|
||||
return .{
|
||||
.gpa = gpa,
|
||||
.input = input,
|
||||
.buffer = Decode.CircularBuffer.init(params.dict_size, mem_limit),
|
||||
.range_decoder = try RangeDecoder.init(input),
|
||||
.decode = try Decode.init(gpa, params.properties),
|
||||
.reader = .{
|
||||
.buffer = buffer,
|
||||
.vtable = &.{
|
||||
.readVec = readVec,
|
||||
.stream = stream,
|
||||
.discard = discard,
|
||||
},
|
||||
.seek = 0,
|
||||
.end = 0,
|
||||
},
|
||||
.err = null,
|
||||
.unpacked_size = params.unpacked_size,
|
||||
};
|
||||
}
|
||||
|
||||
/// Takes ownership of `buffer` which may be resized with `gpa`.
|
||||
///
|
||||
/// LZMA was explicitly designed to take advantage of large heap memory
|
||||
/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
|
||||
/// this API dynamically allocates the dictionary as-needed.
|
||||
pub fn initOptions(
|
||||
input: *Reader,
|
||||
gpa: Allocator,
|
||||
buffer: []u8,
|
||||
options: Decode.Options,
|
||||
mem_limit: usize,
|
||||
) !Decompress {
|
||||
const params = try Decode.Params.readHeader(input, options);
|
||||
return initParams(input, gpa, buffer, params, mem_limit);
|
||||
}
|
||||
|
||||
/// Reclaim ownership of the buffer passed to `init`.
|
||||
pub fn takeBuffer(d: *Decompress) []u8 {
|
||||
const buffer = d.reader.buffer;
|
||||
d.reader.buffer = &.{};
|
||||
return buffer;
|
||||
}
|
||||
|
||||
pub fn deinit(d: *Decompress) void {
|
||||
const gpa = d.gpa;
|
||||
gpa.free(d.reader.buffer);
|
||||
d.buffer.deinit(gpa);
|
||||
d.decode.deinit(gpa);
|
||||
d.* = undefined;
|
||||
}
|
||||
|
||||
fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
|
||||
_ = data;
|
||||
return readIndirect(r);
|
||||
}
|
||||
|
||||
fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
|
||||
_ = w;
|
||||
_ = limit;
|
||||
return readIndirect(r);
|
||||
}
|
||||
|
||||
fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
|
||||
const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
|
||||
_ = d;
|
||||
_ = limit;
|
||||
@panic("TODO");
|
||||
}
|
||||
|
||||
fn readIndirect(r: *Reader) Reader.Error!usize {
|
||||
const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
|
||||
const gpa = d.gpa;
|
||||
var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
|
||||
allocating.writer.end = r.end;
|
||||
defer {
|
||||
r.buffer = allocating.writer.buffer;
|
||||
r.end = allocating.writer.end;
|
||||
}
|
||||
if (d.decode.state == math.maxInt(usize)) return error.EndOfStream;
|
||||
|
||||
process_next: {
|
||||
if (d.unpacked_size) |unpacked_size| {
|
||||
if (d.buffer.len >= unpacked_size) break :process_next;
|
||||
} else if (d.range_decoder.isFinished()) {
|
||||
break :process_next;
|
||||
}
|
||||
var n_read: u64 = 0;
|
||||
switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder, &n_read) catch |err| switch (err) {
|
||||
error.WriteFailed => {
|
||||
d.err = error.OutOfMemory;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
error.EndOfStream => {
|
||||
d.err = error.EndOfStream;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
else => |e| {
|
||||
d.err = e;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
}) {
|
||||
.more => return 0,
|
||||
.finished => break :process_next,
|
||||
}
|
||||
}
|
||||
|
||||
if (d.unpacked_size) |unpacked_size| {
|
||||
if (d.buffer.len != unpacked_size) {
|
||||
d.err = error.DecompressedSizeMismatch;
|
||||
return error.ReadFailed;
|
||||
}
|
||||
}
|
||||
|
||||
d.buffer.finish(&allocating.writer) catch |err| switch (err) {
|
||||
error.WriteFailed => {
|
||||
d.err = error.OutOfMemory;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
};
|
||||
d.decode.state = math.maxInt(usize);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
test {
|
||||
_ = @import("lzma/test.zig");
|
||||
_ = @import("lzma/vec2d.zig");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,379 +0,0 @@
|
|||
const std = @import("../../std.zig");
|
||||
const assert = std.debug.assert;
|
||||
const math = std.math;
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
pub const lzbuffer = @import("decode/lzbuffer.zig");
|
||||
pub const rangecoder = @import("decode/rangecoder.zig");
|
||||
|
||||
const LzCircularBuffer = lzbuffer.LzCircularBuffer;
|
||||
const BitTree = rangecoder.BitTree;
|
||||
const LenDecoder = rangecoder.LenDecoder;
|
||||
const RangeDecoder = rangecoder.RangeDecoder;
|
||||
const Vec2D = @import("vec2d.zig").Vec2D;
|
||||
|
||||
pub const Options = struct {
|
||||
unpacked_size: UnpackedSize = .read_from_header,
|
||||
memlimit: ?usize = null,
|
||||
allow_incomplete: bool = false,
|
||||
};
|
||||
|
||||
pub const UnpackedSize = union(enum) {
|
||||
read_from_header,
|
||||
read_header_but_use_provided: ?u64,
|
||||
use_provided: ?u64,
|
||||
};
|
||||
|
||||
const ProcessingStatus = enum {
|
||||
continue_,
|
||||
finished,
|
||||
};
|
||||
|
||||
pub const Properties = struct {
|
||||
lc: u4,
|
||||
lp: u3,
|
||||
pb: u3,
|
||||
|
||||
fn validate(self: Properties) void {
|
||||
assert(self.lc <= 8);
|
||||
assert(self.lp <= 4);
|
||||
assert(self.pb <= 4);
|
||||
}
|
||||
};
|
||||
|
||||
pub const Params = struct {
|
||||
properties: Properties,
|
||||
dict_size: u32,
|
||||
unpacked_size: ?u64,
|
||||
|
||||
pub fn readHeader(reader: anytype, options: Options) !Params {
|
||||
var props = try reader.readByte();
|
||||
if (props >= 225) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const lc = @as(u4, @intCast(props % 9));
|
||||
props /= 9;
|
||||
const lp = @as(u3, @intCast(props % 5));
|
||||
props /= 5;
|
||||
const pb = @as(u3, @intCast(props));
|
||||
|
||||
const dict_size_provided = try reader.readInt(u32, .little);
|
||||
const dict_size = @max(0x1000, dict_size_provided);
|
||||
|
||||
const unpacked_size = switch (options.unpacked_size) {
|
||||
.read_from_header => blk: {
|
||||
const unpacked_size_provided = try reader.readInt(u64, .little);
|
||||
const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
|
||||
break :blk if (marker_mandatory)
|
||||
null
|
||||
else
|
||||
unpacked_size_provided;
|
||||
},
|
||||
.read_header_but_use_provided => |x| blk: {
|
||||
_ = try reader.readInt(u64, .little);
|
||||
break :blk x;
|
||||
},
|
||||
.use_provided => |x| x,
|
||||
};
|
||||
|
||||
return Params{
|
||||
.properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
|
||||
.dict_size = dict_size,
|
||||
.unpacked_size = unpacked_size,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
pub const DecoderState = struct {
|
||||
lzma_props: Properties,
|
||||
unpacked_size: ?u64,
|
||||
literal_probs: Vec2D(u16),
|
||||
pos_slot_decoder: [4]BitTree(6),
|
||||
align_decoder: BitTree(4),
|
||||
pos_decoders: [115]u16,
|
||||
is_match: [192]u16,
|
||||
is_rep: [12]u16,
|
||||
is_rep_g0: [12]u16,
|
||||
is_rep_g1: [12]u16,
|
||||
is_rep_g2: [12]u16,
|
||||
is_rep_0long: [192]u16,
|
||||
state: usize,
|
||||
rep: [4]usize,
|
||||
len_decoder: LenDecoder,
|
||||
rep_len_decoder: LenDecoder,
|
||||
|
||||
pub fn init(
|
||||
allocator: Allocator,
|
||||
lzma_props: Properties,
|
||||
unpacked_size: ?u64,
|
||||
) !DecoderState {
|
||||
return .{
|
||||
.lzma_props = lzma_props,
|
||||
.unpacked_size = unpacked_size,
|
||||
.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
|
||||
.pos_slot_decoder = @splat(.{}),
|
||||
.align_decoder = .{},
|
||||
.pos_decoders = @splat(0x400),
|
||||
.is_match = @splat(0x400),
|
||||
.is_rep = @splat(0x400),
|
||||
.is_rep_g0 = @splat(0x400),
|
||||
.is_rep_g1 = @splat(0x400),
|
||||
.is_rep_g2 = @splat(0x400),
|
||||
.is_rep_0long = @splat(0x400),
|
||||
.state = 0,
|
||||
.rep = @splat(0),
|
||||
.len_decoder = .{},
|
||||
.rep_len_decoder = .{},
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *DecoderState, allocator: Allocator) void {
|
||||
self.literal_probs.deinit(allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
|
||||
new_props.validate();
|
||||
if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
|
||||
self.literal_probs.fill(0x400);
|
||||
} else {
|
||||
self.literal_probs.deinit(allocator);
|
||||
self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
|
||||
}
|
||||
|
||||
self.lzma_props = new_props;
|
||||
for (&self.pos_slot_decoder) |*t| t.reset();
|
||||
self.align_decoder.reset();
|
||||
self.pos_decoders = @splat(0x400);
|
||||
self.is_match = @splat(0x400);
|
||||
self.is_rep = @splat(0x400);
|
||||
self.is_rep_g0 = @splat(0x400);
|
||||
self.is_rep_g1 = @splat(0x400);
|
||||
self.is_rep_g2 = @splat(0x400);
|
||||
self.is_rep_0long = @splat(0x400);
|
||||
self.state = 0;
|
||||
self.rep = @splat(0);
|
||||
self.len_decoder.reset();
|
||||
self.rep_len_decoder.reset();
|
||||
}
|
||||
|
||||
fn processNextInner(
|
||||
self: *DecoderState,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
update: bool,
|
||||
) !ProcessingStatus {
|
||||
const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
|
||||
|
||||
if (!try decoder.decodeBit(
|
||||
reader,
|
||||
&self.is_match[(self.state << 4) + pos_state],
|
||||
update,
|
||||
)) {
|
||||
const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
|
||||
|
||||
if (update) {
|
||||
try buffer.appendLiteral(allocator, byte, writer);
|
||||
|
||||
self.state = if (self.state < 4)
|
||||
0
|
||||
else if (self.state < 10)
|
||||
self.state - 3
|
||||
else
|
||||
self.state - 6;
|
||||
}
|
||||
return .continue_;
|
||||
}
|
||||
|
||||
var len: usize = undefined;
|
||||
if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
|
||||
if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
|
||||
if (!try decoder.decodeBit(
|
||||
reader,
|
||||
&self.is_rep_0long[(self.state << 4) + pos_state],
|
||||
update,
|
||||
)) {
|
||||
if (update) {
|
||||
self.state = if (self.state < 7) 9 else 11;
|
||||
const dist = self.rep[0] + 1;
|
||||
try buffer.appendLz(allocator, 1, dist, writer);
|
||||
}
|
||||
return .continue_;
|
||||
}
|
||||
} else {
|
||||
const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
|
||||
1
|
||||
else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
|
||||
2
|
||||
else
|
||||
3;
|
||||
if (update) {
|
||||
const dist = self.rep[idx];
|
||||
var i = idx;
|
||||
while (i > 0) : (i -= 1) {
|
||||
self.rep[i] = self.rep[i - 1];
|
||||
}
|
||||
self.rep[0] = dist;
|
||||
}
|
||||
}
|
||||
|
||||
len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
|
||||
|
||||
if (update) {
|
||||
self.state = if (self.state < 7) 8 else 11;
|
||||
}
|
||||
} else {
|
||||
if (update) {
|
||||
self.rep[3] = self.rep[2];
|
||||
self.rep[2] = self.rep[1];
|
||||
self.rep[1] = self.rep[0];
|
||||
}
|
||||
|
||||
len = try self.len_decoder.decode(reader, decoder, pos_state, update);
|
||||
|
||||
if (update) {
|
||||
self.state = if (self.state < 7) 7 else 10;
|
||||
}
|
||||
|
||||
const rep_0 = try self.decodeDistance(reader, decoder, len, update);
|
||||
|
||||
if (update) {
|
||||
self.rep[0] = rep_0;
|
||||
if (self.rep[0] == 0xFFFF_FFFF) {
|
||||
if (decoder.isFinished()) {
|
||||
return .finished;
|
||||
}
|
||||
return error.CorruptInput;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (update) {
|
||||
len += 2;
|
||||
|
||||
const dist = self.rep[0] + 1;
|
||||
try buffer.appendLz(allocator, len, dist, writer);
|
||||
}
|
||||
|
||||
return .continue_;
|
||||
}
|
||||
|
||||
fn processNext(
|
||||
self: *DecoderState,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
) !ProcessingStatus {
|
||||
return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
|
||||
}
|
||||
|
||||
pub fn process(
|
||||
self: *DecoderState,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
) !ProcessingStatus {
|
||||
process_next: {
|
||||
if (self.unpacked_size) |unpacked_size| {
|
||||
if (buffer.len >= unpacked_size) {
|
||||
break :process_next;
|
||||
}
|
||||
} else if (decoder.isFinished()) {
|
||||
break :process_next;
|
||||
}
|
||||
|
||||
switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
|
||||
.continue_ => return .continue_,
|
||||
.finished => break :process_next,
|
||||
}
|
||||
}
|
||||
|
||||
if (self.unpacked_size) |unpacked_size| {
|
||||
if (buffer.len != unpacked_size) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
}
|
||||
|
||||
return .finished;
|
||||
}
|
||||
|
||||
fn decodeLiteral(
|
||||
self: *DecoderState,
|
||||
reader: anytype,
|
||||
buffer: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
update: bool,
|
||||
) !u8 {
|
||||
const def_prev_byte = 0;
|
||||
const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
|
||||
|
||||
var result: usize = 1;
|
||||
const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
|
||||
(prev_byte >> (8 - self.lzma_props.lc));
|
||||
const probs = try self.literal_probs.getMut(lit_state);
|
||||
|
||||
if (self.state >= 7) {
|
||||
var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
|
||||
|
||||
while (result < 0x100) {
|
||||
const match_bit = (match_byte >> 7) & 1;
|
||||
match_byte <<= 1;
|
||||
const bit = @intFromBool(try decoder.decodeBit(
|
||||
reader,
|
||||
&probs[((@as(usize, 1) + match_bit) << 8) + result],
|
||||
update,
|
||||
));
|
||||
result = (result << 1) ^ bit;
|
||||
if (match_bit != bit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (result < 0x100) {
|
||||
result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
|
||||
}
|
||||
|
||||
return @as(u8, @truncate(result - 0x100));
|
||||
}
|
||||
|
||||
fn decodeDistance(
|
||||
self: *DecoderState,
|
||||
reader: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
length: usize,
|
||||
update: bool,
|
||||
) !usize {
|
||||
const len_state = if (length > 3) 3 else length;
|
||||
|
||||
const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
|
||||
if (pos_slot < 4)
|
||||
return pos_slot;
|
||||
|
||||
const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
|
||||
var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
|
||||
|
||||
if (pos_slot < 14) {
|
||||
result += try decoder.parseReverseBitTree(
|
||||
reader,
|
||||
num_direct_bits,
|
||||
&self.pos_decoders,
|
||||
result - pos_slot,
|
||||
update,
|
||||
);
|
||||
} else {
|
||||
result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
|
||||
result += try self.align_decoder.parseReverse(reader, decoder, update);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
const std = @import("../../../std.zig");
|
||||
const math = std.math;
|
||||
const mem = std.mem;
|
||||
const Allocator = std.mem.Allocator;
|
||||
const ArrayListUnmanaged = std.ArrayListUnmanaged;
|
||||
|
||||
/// An accumulating buffer for LZ sequences
|
||||
pub const LzAccumBuffer = struct {
|
||||
/// Buffer
|
||||
buf: ArrayListUnmanaged(u8),
|
||||
|
||||
/// Buffer memory limit
|
||||
memlimit: usize,
|
||||
|
||||
/// Total number of bytes sent through the buffer
|
||||
len: usize,
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub fn init(memlimit: usize) Self {
|
||||
return Self{
|
||||
.buf = .{},
|
||||
.memlimit = memlimit,
|
||||
.len = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
|
||||
try self.buf.append(allocator, byte);
|
||||
self.len += 1;
|
||||
}
|
||||
|
||||
/// Reset the internal dictionary
|
||||
pub fn reset(self: *Self, writer: anytype) !void {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.buf.clearRetainingCapacity();
|
||||
self.len = 0;
|
||||
}
|
||||
|
||||
/// Retrieve the last byte or return a default
|
||||
pub fn lastOr(self: Self, lit: u8) u8 {
|
||||
const buf_len = self.buf.items.len;
|
||||
return if (buf_len == 0)
|
||||
lit
|
||||
else
|
||||
self.buf.items[buf_len - 1];
|
||||
}
|
||||
|
||||
/// Retrieve the n-th last byte
|
||||
pub fn lastN(self: Self, dist: usize) !u8 {
|
||||
const buf_len = self.buf.items.len;
|
||||
if (dist > buf_len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
return self.buf.items[buf_len - dist];
|
||||
}
|
||||
|
||||
/// Append a literal
|
||||
pub fn appendLiteral(
|
||||
self: *Self,
|
||||
allocator: Allocator,
|
||||
lit: u8,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
_ = writer;
|
||||
if (self.len >= self.memlimit) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
try self.buf.append(allocator, lit);
|
||||
self.len += 1;
|
||||
}
|
||||
|
||||
/// Fetch an LZ sequence (length, distance) from inside the buffer
|
||||
pub fn appendLz(
|
||||
self: *Self,
|
||||
allocator: Allocator,
|
||||
len: usize,
|
||||
dist: usize,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
_ = writer;
|
||||
|
||||
const buf_len = self.buf.items.len;
|
||||
if (dist > buf_len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
var offset = buf_len - dist;
|
||||
var i: usize = 0;
|
||||
while (i < len) : (i += 1) {
|
||||
const x = self.buf.items[offset];
|
||||
try self.buf.append(allocator, x);
|
||||
offset += 1;
|
||||
}
|
||||
self.len += len;
|
||||
}
|
||||
|
||||
pub fn finish(self: *Self, writer: anytype) !void {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.buf.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self, allocator: Allocator) void {
|
||||
self.buf.deinit(allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/// A circular buffer for LZ sequences
|
||||
pub const LzCircularBuffer = struct {
|
||||
/// Circular buffer
|
||||
buf: ArrayListUnmanaged(u8),
|
||||
|
||||
/// Length of the buffer
|
||||
dict_size: usize,
|
||||
|
||||
/// Buffer memory limit
|
||||
memlimit: usize,
|
||||
|
||||
/// Current position
|
||||
cursor: usize,
|
||||
|
||||
/// Total number of bytes sent through the buffer
|
||||
len: usize,
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub fn init(dict_size: usize, memlimit: usize) Self {
|
||||
return Self{
|
||||
.buf = .{},
|
||||
.dict_size = dict_size,
|
||||
.memlimit = memlimit,
|
||||
.cursor = 0,
|
||||
.len = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn get(self: Self, index: usize) u8 {
|
||||
return if (0 <= index and index < self.buf.items.len)
|
||||
self.buf.items[index]
|
||||
else
|
||||
0;
|
||||
}
|
||||
|
||||
pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
|
||||
if (index >= self.memlimit) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
try self.buf.ensureTotalCapacity(allocator, index + 1);
|
||||
while (self.buf.items.len < index) {
|
||||
self.buf.appendAssumeCapacity(0);
|
||||
}
|
||||
self.buf.appendAssumeCapacity(value);
|
||||
}
|
||||
|
||||
/// Retrieve the last byte or return a default
|
||||
pub fn lastOr(self: Self, lit: u8) u8 {
|
||||
return if (self.len == 0)
|
||||
lit
|
||||
else
|
||||
self.get((self.dict_size + self.cursor - 1) % self.dict_size);
|
||||
}
|
||||
|
||||
/// Retrieve the n-th last byte
|
||||
pub fn lastN(self: Self, dist: usize) !u8 {
|
||||
if (dist > self.dict_size or dist > self.len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
|
||||
return self.get(offset);
|
||||
}
|
||||
|
||||
/// Append a literal
|
||||
pub fn appendLiteral(
|
||||
self: *Self,
|
||||
allocator: Allocator,
|
||||
lit: u8,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
try self.set(allocator, self.cursor, lit);
|
||||
self.cursor += 1;
|
||||
self.len += 1;
|
||||
|
||||
// Flush the circular buffer to the output
|
||||
if (self.cursor == self.dict_size) {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.cursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch an LZ sequence (length, distance) from inside the buffer
|
||||
pub fn appendLz(
|
||||
self: *Self,
|
||||
allocator: Allocator,
|
||||
len: usize,
|
||||
dist: usize,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
if (dist > self.dict_size or dist > self.len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
|
||||
var i: usize = 0;
|
||||
while (i < len) : (i += 1) {
|
||||
const x = self.get(offset);
|
||||
try self.appendLiteral(allocator, x, writer);
|
||||
offset += 1;
|
||||
if (offset == self.dict_size) {
|
||||
offset = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(self: *Self, writer: anytype) !void {
|
||||
if (self.cursor > 0) {
|
||||
try writer.writeAll(self.buf.items[0..self.cursor]);
|
||||
self.cursor = 0;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self, allocator: Allocator) void {
|
||||
self.buf.deinit(allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
|
@ -1,181 +0,0 @@
|
|||
const std = @import("../../../std.zig");
|
||||
const mem = std.mem;
|
||||
|
||||
pub const RangeDecoder = struct {
|
||||
range: u32,
|
||||
code: u32,
|
||||
|
||||
pub fn init(reader: anytype) !RangeDecoder {
|
||||
const reserved = try reader.readByte();
|
||||
if (reserved != 0) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
return RangeDecoder{
|
||||
.range = 0xFFFF_FFFF,
|
||||
.code = try reader.readInt(u32, .big),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn fromParts(
|
||||
range: u32,
|
||||
code: u32,
|
||||
) RangeDecoder {
|
||||
return .{
|
||||
.range = range,
|
||||
.code = code,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
|
||||
self.range = range;
|
||||
self.code = code;
|
||||
}
|
||||
|
||||
pub inline fn isFinished(self: RangeDecoder) bool {
|
||||
return self.code == 0;
|
||||
}
|
||||
|
||||
inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
|
||||
if (self.range < 0x0100_0000) {
|
||||
self.range <<= 8;
|
||||
self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
|
||||
}
|
||||
}
|
||||
|
||||
inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
|
||||
self.range >>= 1;
|
||||
|
||||
const bit = self.code >= self.range;
|
||||
if (bit)
|
||||
self.code -= self.range;
|
||||
|
||||
try self.normalize(reader);
|
||||
return bit;
|
||||
}
|
||||
|
||||
pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
|
||||
var result: u32 = 0;
|
||||
var i: usize = 0;
|
||||
while (i < count) : (i += 1)
|
||||
result = (result << 1) ^ @intFromBool(try self.getBit(reader));
|
||||
return result;
|
||||
}
|
||||
|
||||
pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
|
||||
const bound = (self.range >> 11) * prob.*;
|
||||
|
||||
if (self.code < bound) {
|
||||
if (update)
|
||||
prob.* += (0x800 - prob.*) >> 5;
|
||||
self.range = bound;
|
||||
|
||||
try self.normalize(reader);
|
||||
return false;
|
||||
} else {
|
||||
if (update)
|
||||
prob.* -= prob.* >> 5;
|
||||
self.code -= bound;
|
||||
self.range -= bound;
|
||||
|
||||
try self.normalize(reader);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
fn parseBitTree(
|
||||
self: *RangeDecoder,
|
||||
reader: anytype,
|
||||
num_bits: u5,
|
||||
probs: []u16,
|
||||
update: bool,
|
||||
) !u32 {
|
||||
var tmp: u32 = 1;
|
||||
var i: @TypeOf(num_bits) = 0;
|
||||
while (i < num_bits) : (i += 1) {
|
||||
const bit = try self.decodeBit(reader, &probs[tmp], update);
|
||||
tmp = (tmp << 1) ^ @intFromBool(bit);
|
||||
}
|
||||
return tmp - (@as(u32, 1) << num_bits);
|
||||
}
|
||||
|
||||
pub fn parseReverseBitTree(
|
||||
self: *RangeDecoder,
|
||||
reader: anytype,
|
||||
num_bits: u5,
|
||||
probs: []u16,
|
||||
offset: usize,
|
||||
update: bool,
|
||||
) !u32 {
|
||||
var result: u32 = 0;
|
||||
var tmp: usize = 1;
|
||||
var i: @TypeOf(num_bits) = 0;
|
||||
while (i < num_bits) : (i += 1) {
|
||||
const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], update));
|
||||
tmp = (tmp << 1) ^ bit;
|
||||
result ^= @as(u32, bit) << i;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
pub fn BitTree(comptime num_bits: usize) type {
|
||||
return struct {
|
||||
probs: [1 << num_bits]u16 = @splat(0x400),
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub fn parse(
|
||||
self: *Self,
|
||||
reader: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
update: bool,
|
||||
) !u32 {
|
||||
return decoder.parseBitTree(reader, num_bits, &self.probs, update);
|
||||
}
|
||||
|
||||
pub fn parseReverse(
|
||||
self: *Self,
|
||||
reader: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
update: bool,
|
||||
) !u32 {
|
||||
return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
|
||||
}
|
||||
|
||||
pub fn reset(self: *Self) void {
|
||||
@memset(&self.probs, 0x400);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub const LenDecoder = struct {
|
||||
choice: u16 = 0x400,
|
||||
choice2: u16 = 0x400,
|
||||
low_coder: [16]BitTree(3) = @splat(.{}),
|
||||
mid_coder: [16]BitTree(3) = @splat(.{}),
|
||||
high_coder: BitTree(8) = .{},
|
||||
|
||||
pub fn decode(
|
||||
self: *LenDecoder,
|
||||
reader: anytype,
|
||||
decoder: *RangeDecoder,
|
||||
pos_state: usize,
|
||||
update: bool,
|
||||
) !usize {
|
||||
if (!try decoder.decodeBit(reader, &self.choice, update)) {
|
||||
return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
|
||||
} else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
|
||||
return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
|
||||
} else {
|
||||
return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(self: *LenDecoder) void {
|
||||
self.choice = 0x400;
|
||||
self.choice2 = 0x400;
|
||||
for (&self.low_coder) |*t| t.reset();
|
||||
for (&self.mid_coder) |*t| t.reset();
|
||||
self.high_coder.reset();
|
||||
}
|
||||
};
|
||||
|
|
@ -1,24 +1,31 @@
|
|||
const std = @import("../../std.zig");
|
||||
const lzma = @import("../lzma.zig");
|
||||
const lzma = std.compress.lzma;
|
||||
|
||||
fn testDecompress(compressed: []const u8) ![]u8 {
|
||||
const allocator = std.testing.allocator;
|
||||
var stream = std.io.fixedBufferStream(compressed);
|
||||
var decompressor = try lzma.decompress(allocator, stream.reader());
|
||||
const gpa = std.testing.allocator;
|
||||
var stream: std.Io.Reader = .fixed(compressed);
|
||||
|
||||
var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
|
||||
defer decompressor.deinit();
|
||||
const reader = decompressor.reader();
|
||||
return reader.readAllAlloc(allocator, std.math.maxInt(usize));
|
||||
return decompressor.reader.allocRemaining(gpa, .unlimited);
|
||||
}
|
||||
|
||||
fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
|
||||
const allocator = std.testing.allocator;
|
||||
const gpa = std.testing.allocator;
|
||||
const decomp = try testDecompress(compressed);
|
||||
defer allocator.free(decomp);
|
||||
defer gpa.free(decomp);
|
||||
try std.testing.expectEqualSlices(u8, expected, decomp);
|
||||
}
|
||||
|
||||
fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
|
||||
return std.testing.expectError(expected, testDecompress(compressed));
|
||||
const gpa = std.testing.allocator;
|
||||
var stream: std.Io.Reader = .fixed(compressed);
|
||||
|
||||
var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
|
||||
defer decompressor.deinit();
|
||||
|
||||
try std.testing.expectError(error.ReadFailed, decompressor.reader.allocRemaining(gpa, .unlimited));
|
||||
try std.testing.expectEqual(expected, decompressor.err orelse return error.TestFailed);
|
||||
}
|
||||
|
||||
test "decompress empty world" {
|
||||
|
|
@ -76,24 +83,26 @@ test "known size with end of payload marker" {
|
|||
|
||||
test "too big uncompressed size in header" {
|
||||
try testDecompressError(
|
||||
error.CorruptInput,
|
||||
error.DecompressedSizeMismatch,
|
||||
@embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
|
||||
);
|
||||
}
|
||||
|
||||
test "too small uncompressed size in header" {
|
||||
try testDecompressError(
|
||||
error.CorruptInput,
|
||||
error.DecompressedSizeMismatch,
|
||||
@embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
|
||||
);
|
||||
}
|
||||
|
||||
test "reading one byte" {
|
||||
const gpa = std.testing.allocator;
|
||||
const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
|
||||
var stream = std.io.fixedBufferStream(compressed);
|
||||
var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());
|
||||
var stream: std.Io.Reader = .fixed(compressed);
|
||||
var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
|
||||
defer decompressor.deinit();
|
||||
|
||||
var buffer = [1]u8{0};
|
||||
_ = try decompressor.read(buffer[0..]);
|
||||
var buffer: [1]u8 = undefined;
|
||||
try decompressor.reader.readSliceAll(&buffer);
|
||||
try std.testing.expectEqual(72, buffer[0]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
const std = @import("../../std.zig");
|
||||
const math = std.math;
|
||||
const mem = std.mem;
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
pub fn Vec2D(comptime T: type) type {
|
||||
return struct {
|
||||
data: []T,
|
||||
cols: usize,
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
|
||||
const len = try math.mul(usize, size[0], size[1]);
|
||||
const data = try allocator.alloc(T, len);
|
||||
@memset(data, value);
|
||||
return Self{
|
||||
.data = data,
|
||||
.cols = size[1],
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self, allocator: Allocator) void {
|
||||
allocator.free(self.data);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn fill(self: *Self, value: T) void {
|
||||
@memset(self.data, value);
|
||||
}
|
||||
|
||||
inline fn _get(self: Self, row: usize) ![]T {
|
||||
const start_row = try math.mul(usize, row, self.cols);
|
||||
const end_row = try math.add(usize, start_row, self.cols);
|
||||
return self.data[start_row..end_row];
|
||||
}
|
||||
|
||||
pub fn get(self: Self, row: usize) ![]const T {
|
||||
return self._get(row);
|
||||
}
|
||||
|
||||
pub fn getMut(self: *Self, row: usize) ![]T {
|
||||
return self._get(row);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
const expectEqualSlices = std.testing.expectEqualSlices;
|
||||
const expectError = std.testing.expectError;
|
||||
|
||||
test "init" {
|
||||
const allocator = testing.allocator;
|
||||
var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
|
||||
defer vec2d.deinit(allocator);
|
||||
|
||||
try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
|
||||
try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
|
||||
}
|
||||
|
||||
test "init overflow" {
|
||||
const allocator = testing.allocator;
|
||||
try expectError(
|
||||
error.Overflow,
|
||||
Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
|
||||
);
|
||||
}
|
||||
|
||||
test "fill" {
|
||||
const allocator = testing.allocator;
|
||||
var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
|
||||
defer vec2d.deinit(allocator);
|
||||
|
||||
vec2d.fill(7);
|
||||
|
||||
try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
|
||||
try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
|
||||
}
|
||||
|
||||
test "get" {
|
||||
var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
|
||||
const vec2d = Vec2D(i32){
|
||||
.data = &data,
|
||||
.cols = 2,
|
||||
};
|
||||
|
||||
try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
|
||||
try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
|
||||
try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
|
||||
try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
|
||||
}
|
||||
|
||||
test "getMut" {
|
||||
var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
|
||||
var vec2d = Vec2D(i32){
|
||||
.data = &data,
|
||||
.cols = 2,
|
||||
};
|
||||
|
||||
const row = try vec2d.getMut(1);
|
||||
row[1] = 9;
|
||||
|
||||
try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
|
||||
// (1, 1) should be 9.
|
||||
try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
|
||||
try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
|
||||
try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
|
||||
}
|
||||
|
||||
test "get multiplication overflow" {
|
||||
const allocator = testing.allocator;
|
||||
var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
|
||||
defer matrix.deinit(allocator);
|
||||
|
||||
const row = (math.maxInt(usize) / 4) + 1;
|
||||
try expectError(error.Overflow, matrix.get(row));
|
||||
try expectError(error.Overflow, matrix.getMut(row));
|
||||
}
|
||||
|
||||
test "get addition overflow" {
|
||||
const allocator = testing.allocator;
|
||||
var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
|
||||
defer matrix.deinit(allocator);
|
||||
|
||||
const row = math.maxInt(usize) / 5;
|
||||
try expectError(error.Overflow, matrix.get(row));
|
||||
try expectError(error.Overflow, matrix.getMut(row));
|
||||
}
|
||||
|
|
@ -1,26 +1,282 @@
|
|||
const std = @import("../std.zig");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const ArrayList = std.ArrayList;
|
||||
const lzma = std.compress.lzma;
|
||||
const Writer = std.Io.Writer;
|
||||
const Reader = std.Io.Reader;
|
||||
|
||||
pub const decode = @import("lzma2/decode.zig");
|
||||
/// An accumulating buffer for LZ sequences
|
||||
pub const AccumBuffer = struct {
|
||||
/// Buffer
|
||||
buf: ArrayList(u8),
|
||||
/// Buffer memory limit
|
||||
memlimit: usize,
|
||||
/// Total number of bytes sent through the buffer
|
||||
len: usize,
|
||||
|
||||
pub fn decompress(
|
||||
pub fn init(memlimit: usize) AccumBuffer {
|
||||
return .{
|
||||
.buf = .{},
|
||||
.memlimit = memlimit,
|
||||
.len = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn appendByte(self: *AccumBuffer, allocator: Allocator, byte: u8) !void {
|
||||
try self.buf.append(allocator, byte);
|
||||
self.len += 1;
|
||||
}
|
||||
|
||||
/// Reset the internal dictionary
|
||||
pub fn reset(self: *AccumBuffer, writer: *Writer) !void {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.buf.clearRetainingCapacity();
|
||||
self.len = 0;
|
||||
}
|
||||
|
||||
/// Retrieve the last byte or return a default
|
||||
pub fn lastOr(self: AccumBuffer, lit: u8) u8 {
|
||||
const buf_len = self.buf.items.len;
|
||||
return if (buf_len == 0)
|
||||
lit
|
||||
else
|
||||
self.buf.items[buf_len - 1];
|
||||
}
|
||||
|
||||
/// Retrieve the n-th last byte
|
||||
pub fn lastN(self: AccumBuffer, dist: usize) !u8 {
|
||||
const buf_len = self.buf.items.len;
|
||||
if (dist > buf_len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
return self.buf.items[buf_len - dist];
|
||||
}
|
||||
|
||||
/// Append a literal
|
||||
pub fn appendLiteral(
|
||||
self: *AccumBuffer,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
var decoder = try decode.Decoder.init(allocator);
|
||||
defer decoder.deinit(allocator);
|
||||
return decoder.decompress(allocator, reader, writer);
|
||||
}
|
||||
lit: u8,
|
||||
writer: *Writer,
|
||||
) !void {
|
||||
_ = writer;
|
||||
if (self.len >= self.memlimit) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
try self.buf.append(allocator, lit);
|
||||
self.len += 1;
|
||||
}
|
||||
|
||||
test {
|
||||
/// Fetch an LZ sequence (length, distance) from inside the buffer
|
||||
pub fn appendLz(
|
||||
self: *AccumBuffer,
|
||||
allocator: Allocator,
|
||||
len: usize,
|
||||
dist: usize,
|
||||
writer: *Writer,
|
||||
) !void {
|
||||
_ = writer;
|
||||
|
||||
const buf_len = self.buf.items.len;
|
||||
if (dist > buf_len) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
var offset = buf_len - dist;
|
||||
var i: usize = 0;
|
||||
while (i < len) : (i += 1) {
|
||||
const x = self.buf.items[offset];
|
||||
try self.buf.append(allocator, x);
|
||||
offset += 1;
|
||||
}
|
||||
self.len += len;
|
||||
}
|
||||
|
||||
pub fn finish(self: *AccumBuffer, writer: *Writer) !void {
|
||||
try writer.writeAll(self.buf.items);
|
||||
self.buf.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *AccumBuffer, allocator: Allocator) void {
|
||||
self.buf.deinit(allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
pub const Decode = struct {
|
||||
lzma_decode: lzma.Decode,
|
||||
|
||||
pub fn init(gpa: Allocator) !Decode {
|
||||
return .{ .lzma_decode = try lzma.Decode.init(gpa, .{ .lc = 0, .lp = 0, .pb = 0 }) };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Decode, gpa: Allocator) void {
|
||||
self.lzma_decode.deinit(gpa);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
/// Returns how many compressed bytes were consumed.
|
||||
pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !u64 {
|
||||
const gpa = allocating.allocator;
|
||||
|
||||
var accum = AccumBuffer.init(std.math.maxInt(usize));
|
||||
defer accum.deinit(gpa);
|
||||
|
||||
var n_read: u64 = 0;
|
||||
|
||||
while (true) {
|
||||
const status = try reader.takeByte();
|
||||
n_read += 1;
|
||||
|
||||
switch (status) {
|
||||
0 => break,
|
||||
1 => n_read += try parseUncompressed(reader, allocating, &accum, true),
|
||||
2 => n_read += try parseUncompressed(reader, allocating, &accum, false),
|
||||
else => n_read += try d.parseLzma(reader, allocating, &accum, status),
|
||||
}
|
||||
}
|
||||
|
||||
try accum.finish(&allocating.writer);
|
||||
return n_read;
|
||||
}
|
||||
|
||||
fn parseLzma(
|
||||
d: *Decode,
|
||||
reader: *Reader,
|
||||
allocating: *Writer.Allocating,
|
||||
accum: *AccumBuffer,
|
||||
status: u8,
|
||||
) !u64 {
|
||||
if (status & 0x80 == 0) return error.CorruptInput;
|
||||
|
||||
const Reset = struct {
|
||||
dict: bool,
|
||||
state: bool,
|
||||
props: bool,
|
||||
};
|
||||
|
||||
const reset: Reset = switch ((status >> 5) & 0x3) {
|
||||
0 => .{
|
||||
.dict = false,
|
||||
.state = false,
|
||||
.props = false,
|
||||
},
|
||||
1 => .{
|
||||
.dict = false,
|
||||
.state = true,
|
||||
.props = false,
|
||||
},
|
||||
2 => .{
|
||||
.dict = false,
|
||||
.state = true,
|
||||
.props = true,
|
||||
},
|
||||
3 => .{
|
||||
.dict = true,
|
||||
.state = true,
|
||||
.props = true,
|
||||
},
|
||||
else => unreachable,
|
||||
};
|
||||
|
||||
var n_read: u64 = 0;
|
||||
|
||||
const unpacked_size = blk: {
|
||||
var tmp: u64 = status & 0x1F;
|
||||
tmp <<= 16;
|
||||
tmp |= try reader.takeInt(u16, .big);
|
||||
n_read += 2;
|
||||
break :blk tmp + 1;
|
||||
};
|
||||
|
||||
const packed_size = blk: {
|
||||
const tmp: u17 = try reader.takeInt(u16, .big);
|
||||
n_read += 2;
|
||||
break :blk tmp + 1;
|
||||
};
|
||||
|
||||
if (reset.dict) try accum.reset(&allocating.writer);
|
||||
|
||||
const ld = &d.lzma_decode;
|
||||
|
||||
if (reset.state) {
|
||||
var new_props = ld.properties;
|
||||
|
||||
if (reset.props) {
|
||||
var props = try reader.takeByte();
|
||||
n_read += 1;
|
||||
if (props >= 225) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const lc = @as(u4, @intCast(props % 9));
|
||||
props /= 9;
|
||||
const lp = @as(u3, @intCast(props % 5));
|
||||
props /= 5;
|
||||
const pb = @as(u3, @intCast(props));
|
||||
|
||||
if (lc + lp > 4) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
new_props = .{ .lc = lc, .lp = lp, .pb = pb };
|
||||
}
|
||||
|
||||
try ld.resetState(allocating.allocator, new_props);
|
||||
}
|
||||
|
||||
const expected_unpacked_size = accum.len + unpacked_size;
|
||||
const start_count = n_read;
|
||||
var range_decoder = try lzma.RangeDecoder.initCounting(reader, &n_read);
|
||||
|
||||
while (true) {
|
||||
if (accum.len >= expected_unpacked_size) break;
|
||||
if (range_decoder.isFinished()) break;
|
||||
switch (try ld.process(reader, allocating, accum, &range_decoder, &n_read)) {
|
||||
.more => continue,
|
||||
.finished => break,
|
||||
}
|
||||
}
|
||||
if (accum.len != expected_unpacked_size) return error.DecompressedSizeMismatch;
|
||||
if (n_read - start_count != packed_size) return error.CompressedSizeMismatch;
|
||||
|
||||
return n_read;
|
||||
}
|
||||
|
||||
fn parseUncompressed(
|
||||
reader: *Reader,
|
||||
allocating: *Writer.Allocating,
|
||||
accum: *AccumBuffer,
|
||||
reset_dict: bool,
|
||||
) !usize {
|
||||
const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
|
||||
|
||||
if (reset_dict) try accum.reset(&allocating.writer);
|
||||
|
||||
const gpa = allocating.allocator;
|
||||
|
||||
for (0..unpacked_size) |_| {
|
||||
try accum.appendByte(gpa, try reader.takeByte());
|
||||
}
|
||||
return 2 + unpacked_size;
|
||||
}
|
||||
};
|
||||
|
||||
test "decompress hello world stream" {
|
||||
const expected = "Hello\nWorld!\n";
|
||||
const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
|
||||
|
||||
const allocator = std.testing.allocator;
|
||||
var decomp = std.array_list.Managed(u8).init(allocator);
|
||||
defer decomp.deinit();
|
||||
var stream = std.io.fixedBufferStream(compressed);
|
||||
try decompress(allocator, stream.reader(), decomp.writer());
|
||||
try std.testing.expectEqualSlices(u8, expected, decomp.items);
|
||||
const gpa = std.testing.allocator;
|
||||
|
||||
var decode = try Decode.init(gpa);
|
||||
defer decode.deinit(gpa);
|
||||
|
||||
var stream: std.Io.Reader = .fixed(compressed);
|
||||
var result: std.Io.Writer.Allocating = .init(gpa);
|
||||
defer result.deinit();
|
||||
|
||||
const n_read = try decode.decompress(&stream, &result);
|
||||
try std.testing.expectEqual(compressed.len, n_read);
|
||||
try std.testing.expectEqualStrings(expected, result.written());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
const std = @import("../../std.zig");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const lzma = @import("../lzma.zig");
|
||||
const DecoderState = lzma.decode.DecoderState;
|
||||
const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
|
||||
const Properties = lzma.decode.Properties;
|
||||
const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
|
||||
|
||||
pub const Decoder = struct {
|
||||
lzma_state: DecoderState,
|
||||
|
||||
pub fn init(allocator: Allocator) !Decoder {
|
||||
return Decoder{
|
||||
.lzma_state = try DecoderState.init(
|
||||
allocator,
|
||||
Properties{
|
||||
.lc = 0,
|
||||
.lp = 0,
|
||||
.pb = 0,
|
||||
},
|
||||
null,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Decoder, allocator: Allocator) void {
|
||||
self.lzma_state.deinit(allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn decompress(
|
||||
self: *Decoder,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
) !void {
|
||||
var accum = LzAccumBuffer.init(std.math.maxInt(usize));
|
||||
defer accum.deinit(allocator);
|
||||
|
||||
while (true) {
|
||||
const status = try reader.readByte();
|
||||
|
||||
switch (status) {
|
||||
0 => break,
|
||||
1 => try parseUncompressed(allocator, reader, writer, &accum, true),
|
||||
2 => try parseUncompressed(allocator, reader, writer, &accum, false),
|
||||
else => try self.parseLzma(allocator, reader, writer, &accum, status),
|
||||
}
|
||||
}
|
||||
|
||||
try accum.finish(writer);
|
||||
}
|
||||
|
||||
fn parseLzma(
|
||||
self: *Decoder,
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
accum: *LzAccumBuffer,
|
||||
status: u8,
|
||||
) !void {
|
||||
if (status & 0x80 == 0) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const Reset = struct {
|
||||
dict: bool,
|
||||
state: bool,
|
||||
props: bool,
|
||||
};
|
||||
|
||||
const reset = switch ((status >> 5) & 0x3) {
|
||||
0 => Reset{
|
||||
.dict = false,
|
||||
.state = false,
|
||||
.props = false,
|
||||
},
|
||||
1 => Reset{
|
||||
.dict = false,
|
||||
.state = true,
|
||||
.props = false,
|
||||
},
|
||||
2 => Reset{
|
||||
.dict = false,
|
||||
.state = true,
|
||||
.props = true,
|
||||
},
|
||||
3 => Reset{
|
||||
.dict = true,
|
||||
.state = true,
|
||||
.props = true,
|
||||
},
|
||||
else => unreachable,
|
||||
};
|
||||
|
||||
const unpacked_size = blk: {
|
||||
var tmp: u64 = status & 0x1F;
|
||||
tmp <<= 16;
|
||||
tmp |= try reader.readInt(u16, .big);
|
||||
break :blk tmp + 1;
|
||||
};
|
||||
|
||||
const packed_size = blk: {
|
||||
const tmp: u17 = try reader.readInt(u16, .big);
|
||||
break :blk tmp + 1;
|
||||
};
|
||||
|
||||
if (reset.dict) {
|
||||
try accum.reset(writer);
|
||||
}
|
||||
|
||||
if (reset.state) {
|
||||
var new_props = self.lzma_state.lzma_props;
|
||||
|
||||
if (reset.props) {
|
||||
var props = try reader.readByte();
|
||||
if (props >= 225) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const lc = @as(u4, @intCast(props % 9));
|
||||
props /= 9;
|
||||
const lp = @as(u3, @intCast(props % 5));
|
||||
props /= 5;
|
||||
const pb = @as(u3, @intCast(props));
|
||||
|
||||
if (lc + lp > 4) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
|
||||
}
|
||||
|
||||
try self.lzma_state.resetState(allocator, new_props);
|
||||
}
|
||||
|
||||
self.lzma_state.unpacked_size = unpacked_size + accum.len;
|
||||
|
||||
var counter = std.io.countingReader(reader);
|
||||
const counter_reader = counter.reader();
|
||||
|
||||
var rangecoder = try RangeDecoder.init(counter_reader);
|
||||
while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
|
||||
|
||||
if (counter.bytes_read != packed_size) {
|
||||
return error.CorruptInput;
|
||||
}
|
||||
}
|
||||
|
||||
fn parseUncompressed(
|
||||
allocator: Allocator,
|
||||
reader: anytype,
|
||||
writer: anytype,
|
||||
accum: *LzAccumBuffer,
|
||||
reset_dict: bool,
|
||||
) !void {
|
||||
const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
|
||||
|
||||
if (reset_dict) {
|
||||
try accum.reset(writer);
|
||||
}
|
||||
|
||||
var i: @TypeOf(unpacked_size) = 0;
|
||||
while (i < unpacked_size) : (i += 1) {
|
||||
try accum.appendByte(allocator, try reader.readByte());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -1,165 +1,4 @@
|
|||
const std = @import("std");
|
||||
const block = @import("xz/block.zig");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const Crc32 = std.hash.Crc32;
|
||||
|
||||
pub const Check = enum(u4) {
|
||||
none = 0x00,
|
||||
crc32 = 0x01,
|
||||
crc64 = 0x04,
|
||||
sha256 = 0x0A,
|
||||
_,
|
||||
};
|
||||
|
||||
fn readStreamFlags(reader: anytype, check: *Check) !void {
|
||||
const reserved1 = try reader.readByte();
|
||||
if (reserved1 != 0) return error.CorruptInput;
|
||||
const byte = try reader.readByte();
|
||||
if ((byte >> 4) != 0) return error.CorruptInput;
|
||||
check.* = @enumFromInt(@as(u4, @truncate(byte)));
|
||||
}
|
||||
|
||||
pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
|
||||
return Decompress(@TypeOf(reader)).init(allocator, reader);
|
||||
}
|
||||
|
||||
pub fn Decompress(comptime ReaderType: type) type {
|
||||
return struct {
|
||||
const Self = @This();
|
||||
|
||||
pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
|
||||
pub const Reader = std.io.GenericReader(*Self, Error, read);
|
||||
|
||||
allocator: Allocator,
|
||||
block_decoder: block.Decoder(ReaderType),
|
||||
in_reader: ReaderType,
|
||||
|
||||
fn init(allocator: Allocator, source: ReaderType) !Self {
|
||||
const magic = try source.readBytesNoEof(6);
|
||||
if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
|
||||
return error.BadHeader;
|
||||
|
||||
var check: Check = undefined;
|
||||
const hash_a = blk: {
|
||||
var hasher = hashedReader(source, Crc32.init());
|
||||
try readStreamFlags(hasher.reader(), &check);
|
||||
break :blk hasher.hasher.final();
|
||||
};
|
||||
|
||||
const hash_b = try source.readInt(u32, .little);
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
|
||||
return Self{
|
||||
.allocator = allocator,
|
||||
.block_decoder = try block.decoder(allocator, source, check),
|
||||
.in_reader = source,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.block_decoder.deinit();
|
||||
}
|
||||
|
||||
pub fn reader(self: *Self) Reader {
|
||||
return .{ .context = self };
|
||||
}
|
||||
|
||||
pub fn read(self: *Self, buffer: []u8) Error!usize {
|
||||
if (buffer.len == 0)
|
||||
return 0;
|
||||
|
||||
const r = try self.block_decoder.read(buffer);
|
||||
if (r != 0)
|
||||
return r;
|
||||
|
||||
const index_size = blk: {
|
||||
var hasher = hashedReader(self.in_reader, Crc32.init());
|
||||
hasher.hasher.update(&[1]u8{0x00});
|
||||
|
||||
var counter = std.io.countingReader(hasher.reader());
|
||||
counter.bytes_read += 1;
|
||||
|
||||
const counting_reader = counter.reader();
|
||||
|
||||
const record_count = try std.leb.readUleb128(u64, counting_reader);
|
||||
if (record_count != self.block_decoder.block_count)
|
||||
return error.CorruptInput;
|
||||
|
||||
var i: usize = 0;
|
||||
while (i < record_count) : (i += 1) {
|
||||
// TODO: validate records
|
||||
_ = try std.leb.readUleb128(u64, counting_reader);
|
||||
_ = try std.leb.readUleb128(u64, counting_reader);
|
||||
}
|
||||
|
||||
while (counter.bytes_read % 4 != 0) {
|
||||
if (try counting_reader.readByte() != 0)
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const hash_a = hasher.hasher.final();
|
||||
const hash_b = try counting_reader.readInt(u32, .little);
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
|
||||
break :blk counter.bytes_read;
|
||||
};
|
||||
|
||||
const hash_a = try self.in_reader.readInt(u32, .little);
|
||||
|
||||
const hash_b = blk: {
|
||||
var hasher = hashedReader(self.in_reader, Crc32.init());
|
||||
const hashed_reader = hasher.reader();
|
||||
|
||||
const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
|
||||
if (backward_size != index_size)
|
||||
return error.CorruptInput;
|
||||
|
||||
var check: Check = undefined;
|
||||
try readStreamFlags(hashed_reader, &check);
|
||||
|
||||
break :blk hasher.hasher.final();
|
||||
};
|
||||
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
|
||||
const magic = try self.in_reader.readBytesNoEof(2);
|
||||
if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
|
||||
return error.CorruptInput;
|
||||
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn HashedReader(ReaderType: type, HasherType: type) type {
|
||||
return struct {
|
||||
child_reader: ReaderType,
|
||||
hasher: HasherType,
|
||||
|
||||
pub const Error = ReaderType.Error;
|
||||
pub const Reader = std.io.GenericReader(*@This(), Error, read);
|
||||
|
||||
pub fn read(self: *@This(), buf: []u8) Error!usize {
|
||||
const amt = try self.child_reader.read(buf);
|
||||
self.hasher.update(buf[0..amt]);
|
||||
return amt;
|
||||
}
|
||||
|
||||
pub fn reader(self: *@This()) Reader {
|
||||
return .{ .context = self };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn hashedReader(
|
||||
reader: anytype,
|
||||
hasher: anytype,
|
||||
) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
|
||||
return .{ .child_reader = reader, .hasher = hasher };
|
||||
}
|
||||
pub const Decompress = @import("xz/Decompress.zig");
|
||||
|
||||
test {
|
||||
_ = @import("xz/test.zig");
|
||||
|
|
|
|||
319
lib/std/compress/xz/Decompress.zig
Normal file
319
lib/std/compress/xz/Decompress.zig
Normal file
|
|
@ -0,0 +1,319 @@
|
|||
const Decompress = @This();
|
||||
const std = @import("../../std.zig");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const ArrayList = std.ArrayList;
|
||||
const Crc32 = std.hash.Crc32;
|
||||
const Crc64 = std.hash.crc.Crc64Xz;
|
||||
const Sha256 = std.crypto.hash.sha2.Sha256;
|
||||
const lzma2 = std.compress.lzma2;
|
||||
const Writer = std.Io.Writer;
|
||||
const Reader = std.Io.Reader;
|
||||
const assert = std.debug.assert;
|
||||
|
||||
/// Underlying compressed data stream to pull bytes from.
|
||||
input: *Reader,
|
||||
/// Uncompressed bytes output by this stream implementation.
|
||||
reader: Reader,
|
||||
gpa: Allocator,
|
||||
check: Check,
|
||||
block_count: usize,
|
||||
err: ?Error,
|
||||
|
||||
pub const Error = error{
|
||||
ReadFailed,
|
||||
OutOfMemory,
|
||||
CorruptInput,
|
||||
EndOfStream,
|
||||
WrongChecksum,
|
||||
Unsupported,
|
||||
Overflow,
|
||||
InvalidRangeCode,
|
||||
DecompressedSizeMismatch,
|
||||
CompressedSizeMismatch,
|
||||
};
|
||||
|
||||
pub const Check = enum(u4) {
|
||||
none = 0x00,
|
||||
crc32 = 0x01,
|
||||
crc64 = 0x04,
|
||||
sha256 = 0x0A,
|
||||
_,
|
||||
};
|
||||
|
||||
pub const StreamFlags = packed struct(u16) {
|
||||
null: u8 = 0,
|
||||
check: Check,
|
||||
reserved: u4 = 0,
|
||||
};
|
||||
|
||||
pub const InitError = error{
|
||||
NotXzStream,
|
||||
WrongChecksum,
|
||||
};
|
||||
|
||||
/// XZ uses a series of LZMA2 blocks which each specify a dictionary size
|
||||
/// anywhere from 4K to 4G. Thus, this API dynamically allocates the dictionary
|
||||
/// as-needed.
|
||||
pub fn init(
|
||||
input: *Reader,
|
||||
gpa: Allocator,
|
||||
/// Decompress takes ownership of this buffer and resizes it with `gpa`.
|
||||
buffer: []u8,
|
||||
) !Decompress {
|
||||
const magic = try input.takeArray(6);
|
||||
if (!std.mem.eql(u8, magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
|
||||
return error.NotXzStream;
|
||||
|
||||
const computed_checksum = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
|
||||
const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
|
||||
const stored_hash = try input.takeInt(u32, .little);
|
||||
if (computed_checksum != stored_hash) return error.WrongChecksum;
|
||||
|
||||
return .{
|
||||
.input = input,
|
||||
.reader = .{
|
||||
.vtable = &.{
|
||||
.stream = stream,
|
||||
.readVec = readVec,
|
||||
.discard = discard,
|
||||
},
|
||||
.buffer = buffer,
|
||||
.seek = 0,
|
||||
.end = 0,
|
||||
},
|
||||
.gpa = gpa,
|
||||
.check = stream_flags.check,
|
||||
.block_count = 0,
|
||||
.err = null,
|
||||
};
|
||||
}
|
||||
|
||||
/// Reclaim ownership of the buffer passed to `init`.
|
||||
pub fn takeBuffer(d: *Decompress) []u8 {
|
||||
const buffer = d.reader.buffer;
|
||||
d.reader.buffer = &.{};
|
||||
return buffer;
|
||||
}
|
||||
|
||||
pub fn deinit(d: *Decompress) void {
|
||||
const gpa = d.gpa;
|
||||
gpa.free(d.reader.buffer);
|
||||
d.* = undefined;
|
||||
}
|
||||
|
||||
fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
|
||||
_ = data;
|
||||
return readIndirect(r);
|
||||
}
|
||||
|
||||
fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
|
||||
_ = w;
|
||||
_ = limit;
|
||||
return readIndirect(r);
|
||||
}
|
||||
|
||||
fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
|
||||
const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
|
||||
_ = d;
|
||||
_ = limit;
|
||||
@panic("TODO");
|
||||
}
|
||||
|
||||
fn readIndirect(r: *Reader) Reader.Error!usize {
|
||||
const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
|
||||
const gpa = d.gpa;
|
||||
const input = d.input;
|
||||
|
||||
var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
|
||||
allocating.writer.end = r.end;
|
||||
defer {
|
||||
r.buffer = allocating.writer.buffer;
|
||||
r.end = allocating.writer.end;
|
||||
}
|
||||
|
||||
if (d.err != null) return error.ReadFailed;
|
||||
if (d.block_count == std.math.maxInt(usize)) return error.EndOfStream;
|
||||
|
||||
readBlock(input, &allocating) catch |err| switch (err) {
|
||||
error.WriteFailed => {
|
||||
d.err = error.OutOfMemory;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
error.SuccessfulEndOfStream => {
|
||||
finish(d) catch |finish_err| {
|
||||
d.err = finish_err;
|
||||
return error.ReadFailed;
|
||||
};
|
||||
d.block_count = std.math.maxInt(usize);
|
||||
return error.EndOfStream;
|
||||
},
|
||||
else => |e| {
|
||||
d.err = e;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
};
|
||||
switch (d.check) {
|
||||
.none => {},
|
||||
.crc32 => {
|
||||
const declared_checksum = try input.takeInt(u32, .little);
|
||||
// TODO
|
||||
//const hash_a = Crc32.hash(unpacked_bytes);
|
||||
//if (hash_a != hash_b) return error.WrongChecksum;
|
||||
_ = declared_checksum;
|
||||
},
|
||||
.crc64 => {
|
||||
const declared_checksum = try input.takeInt(u64, .little);
|
||||
// TODO
|
||||
//const hash_a = Crc64.hash(unpacked_bytes);
|
||||
//if (hash_a != hash_b) return error.WrongChecksum;
|
||||
_ = declared_checksum;
|
||||
},
|
||||
.sha256 => {
|
||||
const declared_hash = try input.take(Sha256.digest_length);
|
||||
// TODO
|
||||
//var hash_a: [Sha256.digest_length]u8 = undefined;
|
||||
//Sha256.hash(unpacked_bytes, &hash_a, .{});
|
||||
//if (!std.mem.eql(u8, &hash_a, &hash_b))
|
||||
// return error.WrongChecksum;
|
||||
_ = declared_hash;
|
||||
},
|
||||
else => {
|
||||
d.err = error.Unsupported;
|
||||
return error.ReadFailed;
|
||||
},
|
||||
}
|
||||
d.block_count += 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
|
||||
var packed_size: ?u64 = null;
|
||||
var unpacked_size: ?u64 = null;
|
||||
|
||||
const header_size = h: {
|
||||
// Read the block header via peeking so that we can hash the whole thing too.
|
||||
const first_byte: usize = try input.peekByte();
|
||||
if (first_byte == 0) return error.SuccessfulEndOfStream;
|
||||
|
||||
const declared_header_size = first_byte * 4;
|
||||
try input.fill(declared_header_size);
|
||||
const header_seek_start = input.seek;
|
||||
input.toss(1);
|
||||
|
||||
const Flags = packed struct(u8) {
|
||||
last_filter_index: u2,
|
||||
reserved: u4,
|
||||
has_packed_size: bool,
|
||||
has_unpacked_size: bool,
|
||||
};
|
||||
const flags = try input.takeStruct(Flags, .little);
|
||||
|
||||
const filter_count = @as(u3, flags.last_filter_index) + 1;
|
||||
if (filter_count > 1) return error.Unsupported;
|
||||
|
||||
if (flags.has_packed_size) packed_size = try input.takeLeb128(u64);
|
||||
if (flags.has_unpacked_size) unpacked_size = try input.takeLeb128(u64);
|
||||
|
||||
const FilterId = enum(u64) {
|
||||
lzma2 = 0x21,
|
||||
_,
|
||||
};
|
||||
|
||||
const filter_id: FilterId = @enumFromInt(try input.takeLeb128(u64));
|
||||
if (filter_id != .lzma2) return error.Unsupported;
|
||||
|
||||
const properties_size = try input.takeLeb128(u64);
|
||||
if (properties_size != 1) return error.CorruptInput;
|
||||
// TODO: use filter properties
|
||||
_ = try input.takeByte();
|
||||
|
||||
const actual_header_size = input.seek - header_seek_start;
|
||||
if (actual_header_size > declared_header_size) return error.CorruptInput;
|
||||
const remaining_bytes = declared_header_size - actual_header_size;
|
||||
for (0..remaining_bytes) |_| {
|
||||
if (try input.takeByte() != 0) return error.CorruptInput;
|
||||
}
|
||||
|
||||
const header_slice = input.buffer[header_seek_start..][0..declared_header_size];
|
||||
const computed_checksum = Crc32.hash(header_slice);
|
||||
const declared_checksum = try input.takeInt(u32, .little);
|
||||
if (computed_checksum != declared_checksum) return error.WrongChecksum;
|
||||
break :h declared_header_size;
|
||||
};
|
||||
|
||||
// Compressed Data
|
||||
|
||||
var lzma2_decode = try lzma2.Decode.init(allocating.allocator);
|
||||
defer lzma2_decode.deinit(allocating.allocator);
|
||||
const before_size = allocating.writer.end;
|
||||
const packed_bytes_read = try lzma2_decode.decompress(input, allocating);
|
||||
const unpacked_bytes = allocating.writer.end - before_size;
|
||||
|
||||
if (packed_size) |s| {
|
||||
if (s != packed_bytes_read) return error.CorruptInput;
|
||||
}
|
||||
|
||||
if (unpacked_size) |s| {
|
||||
if (s != unpacked_bytes) return error.CorruptInput;
|
||||
}
|
||||
|
||||
// Block Padding
|
||||
const block_counter = header_size + packed_bytes_read;
|
||||
const padding = try input.take(@intCast((4 - (block_counter % 4)) % 4));
|
||||
for (padding) |byte| {
|
||||
if (byte != 0) return error.CorruptInput;
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(d: *Decompress) !void {
|
||||
const input = d.input;
|
||||
const index_size = blk: {
|
||||
// Assume that we already peeked a zero in readBlock().
|
||||
assert(input.buffered()[0] == 0);
|
||||
var input_counter: u64 = 1;
|
||||
var checksum: Crc32 = .init();
|
||||
checksum.update(&.{0});
|
||||
input.toss(1);
|
||||
|
||||
const record_count = try countLeb128(input, u64, &input_counter, &checksum);
|
||||
if (record_count != d.block_count)
|
||||
return error.CorruptInput;
|
||||
|
||||
for (0..@intCast(record_count)) |_| {
|
||||
// TODO: validate records
|
||||
_ = try countLeb128(input, u64, &input_counter, &checksum);
|
||||
_ = try countLeb128(input, u64, &input_counter, &checksum);
|
||||
}
|
||||
|
||||
const padding = try input.take(@intCast((4 - (input_counter % 4)) % 4));
|
||||
for (padding) |byte| {
|
||||
if (byte != 0) return error.CorruptInput;
|
||||
}
|
||||
checksum.update(padding);
|
||||
|
||||
const declared_checksum = try input.takeInt(u32, .little);
|
||||
const computed_checksum = checksum.final();
|
||||
if (computed_checksum != declared_checksum) return error.WrongChecksum;
|
||||
|
||||
break :blk input_counter + padding.len + 4;
|
||||
};
|
||||
|
||||
const declared_checksum = try input.takeInt(u32, .little);
|
||||
const computed_checksum = Crc32.hash(try input.peek(4 + @sizeOf(StreamFlags)));
|
||||
if (declared_checksum != computed_checksum) return error.WrongChecksum;
|
||||
const backward_size = (@as(u64, try input.takeInt(u32, .little)) + 1) * 4;
|
||||
if (backward_size != index_size) return error.CorruptInput;
|
||||
input.toss(@sizeOf(StreamFlags));
|
||||
if (!std.mem.eql(u8, try input.takeArray(2), &.{ 'Y', 'Z' }))
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
fn countLeb128(reader: *Reader, comptime T: type, counter: *u64, hasher: *Crc32) !T {
|
||||
try reader.fill(8);
|
||||
const start = reader.seek;
|
||||
const result = try reader.takeLeb128(T);
|
||||
const read_slice = reader.buffer[start..reader.seek];
|
||||
hasher.update(read_slice);
|
||||
counter.* += read_slice.len;
|
||||
return result;
|
||||
}
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
const std = @import("../../std.zig");
|
||||
const lzma2 = std.compress.lzma2;
|
||||
const Allocator = std.mem.Allocator;
|
||||
const ArrayListUnmanaged = std.ArrayListUnmanaged;
|
||||
const Crc32 = std.hash.Crc32;
|
||||
const Crc64 = std.hash.crc.Crc64Xz;
|
||||
const Sha256 = std.crypto.hash.sha2.Sha256;
|
||||
const xz = std.compress.xz;
|
||||
|
||||
const DecodeError = error{
|
||||
CorruptInput,
|
||||
EndOfStream,
|
||||
EndOfStreamWithNoError,
|
||||
WrongChecksum,
|
||||
Unsupported,
|
||||
Overflow,
|
||||
};
|
||||
|
||||
pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
|
||||
return Decoder(@TypeOf(reader)).init(allocator, reader, check);
|
||||
}
|
||||
|
||||
pub fn Decoder(comptime ReaderType: type) type {
|
||||
return struct {
|
||||
const Self = @This();
|
||||
pub const Error =
|
||||
ReaderType.Error ||
|
||||
DecodeError ||
|
||||
Allocator.Error;
|
||||
pub const Reader = std.io.GenericReader(*Self, Error, read);
|
||||
|
||||
allocator: Allocator,
|
||||
inner_reader: ReaderType,
|
||||
check: xz.Check,
|
||||
err: ?Error,
|
||||
to_read: ArrayListUnmanaged(u8),
|
||||
read_pos: usize,
|
||||
block_count: usize,
|
||||
|
||||
fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
|
||||
return Self{
|
||||
.allocator = allocator,
|
||||
.inner_reader = in_reader,
|
||||
.check = check,
|
||||
.err = null,
|
||||
.to_read = .{},
|
||||
.read_pos = 0,
|
||||
.block_count = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.to_read.deinit(self.allocator);
|
||||
}
|
||||
|
||||
pub fn reader(self: *Self) Reader {
|
||||
return .{ .context = self };
|
||||
}
|
||||
|
||||
pub fn read(self: *Self, output: []u8) Error!usize {
|
||||
while (true) {
|
||||
const unread_len = self.to_read.items.len - self.read_pos;
|
||||
if (unread_len > 0) {
|
||||
const n = @min(unread_len, output.len);
|
||||
@memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
|
||||
self.read_pos += n;
|
||||
return n;
|
||||
}
|
||||
if (self.err) |e| {
|
||||
if (e == DecodeError.EndOfStreamWithNoError) {
|
||||
return 0;
|
||||
}
|
||||
return e;
|
||||
}
|
||||
if (self.read_pos > 0) {
|
||||
self.to_read.shrinkRetainingCapacity(0);
|
||||
self.read_pos = 0;
|
||||
}
|
||||
self.readBlock() catch |e| {
|
||||
self.err = e;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn readBlock(self: *Self) Error!void {
|
||||
var block_counter = std.io.countingReader(self.inner_reader);
|
||||
const block_reader = block_counter.reader();
|
||||
|
||||
var packed_size: ?u64 = null;
|
||||
var unpacked_size: ?u64 = null;
|
||||
|
||||
// Block Header
|
||||
{
|
||||
var header_hasher = xz.hashedReader(block_reader, Crc32.init());
|
||||
const header_reader = header_hasher.reader();
|
||||
|
||||
const header_size = @as(u64, try header_reader.readByte()) * 4;
|
||||
if (header_size == 0)
|
||||
return error.EndOfStreamWithNoError;
|
||||
|
||||
const Flags = packed struct(u8) {
|
||||
last_filter_index: u2,
|
||||
reserved: u4,
|
||||
has_packed_size: bool,
|
||||
has_unpacked_size: bool,
|
||||
};
|
||||
|
||||
const flags = @as(Flags, @bitCast(try header_reader.readByte()));
|
||||
const filter_count = @as(u3, flags.last_filter_index) + 1;
|
||||
if (filter_count > 1)
|
||||
return error.Unsupported;
|
||||
|
||||
if (flags.has_packed_size)
|
||||
packed_size = try std.leb.readUleb128(u64, header_reader);
|
||||
|
||||
if (flags.has_unpacked_size)
|
||||
unpacked_size = try std.leb.readUleb128(u64, header_reader);
|
||||
|
||||
const FilterId = enum(u64) {
|
||||
lzma2 = 0x21,
|
||||
_,
|
||||
};
|
||||
|
||||
const filter_id = @as(
|
||||
FilterId,
|
||||
@enumFromInt(try std.leb.readUleb128(u64, header_reader)),
|
||||
);
|
||||
|
||||
if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
|
||||
return error.CorruptInput;
|
||||
|
||||
if (filter_id != .lzma2)
|
||||
return error.Unsupported;
|
||||
|
||||
const properties_size = try std.leb.readUleb128(u64, header_reader);
|
||||
if (properties_size != 1)
|
||||
return error.CorruptInput;
|
||||
|
||||
// TODO: use filter properties
|
||||
_ = try header_reader.readByte();
|
||||
|
||||
while (block_counter.bytes_read != header_size) {
|
||||
if (try header_reader.readByte() != 0)
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const hash_a = header_hasher.hasher.final();
|
||||
const hash_b = try header_reader.readInt(u32, .little);
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
}
|
||||
|
||||
// Compressed Data
|
||||
var packed_counter = std.io.countingReader(block_reader);
|
||||
try lzma2.decompress(
|
||||
self.allocator,
|
||||
packed_counter.reader(),
|
||||
self.to_read.writer(self.allocator),
|
||||
);
|
||||
|
||||
if (packed_size) |s| {
|
||||
if (s != packed_counter.bytes_read)
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
const unpacked_bytes = self.to_read.items;
|
||||
if (unpacked_size) |s| {
|
||||
if (s != unpacked_bytes.len)
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
// Block Padding
|
||||
while (block_counter.bytes_read % 4 != 0) {
|
||||
if (try block_reader.readByte() != 0)
|
||||
return error.CorruptInput;
|
||||
}
|
||||
|
||||
switch (self.check) {
|
||||
.none => {},
|
||||
.crc32 => {
|
||||
const hash_a = Crc32.hash(unpacked_bytes);
|
||||
const hash_b = try self.inner_reader.readInt(u32, .little);
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
},
|
||||
.crc64 => {
|
||||
const hash_a = Crc64.hash(unpacked_bytes);
|
||||
const hash_b = try self.inner_reader.readInt(u64, .little);
|
||||
if (hash_a != hash_b)
|
||||
return error.WrongChecksum;
|
||||
},
|
||||
.sha256 => {
|
||||
var hash_a: [Sha256.digest_length]u8 = undefined;
|
||||
Sha256.hash(unpacked_bytes, &hash_a, .{});
|
||||
|
||||
var hash_b: [Sha256.digest_length]u8 = undefined;
|
||||
try self.inner_reader.readNoEof(&hash_b);
|
||||
|
||||
if (!std.mem.eql(u8, &hash_a, &hash_b))
|
||||
return error.WrongChecksum;
|
||||
},
|
||||
else => return error.Unsupported,
|
||||
}
|
||||
|
||||
self.block_count += 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -3,48 +3,79 @@ const testing = std.testing;
|
|||
const xz = std.compress.xz;
|
||||
|
||||
fn decompress(data: []const u8) ![]u8 {
|
||||
var in_stream = std.io.fixedBufferStream(data);
|
||||
const gpa = testing.allocator;
|
||||
|
||||
var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());
|
||||
var in_stream: std.Io.Reader = .fixed(data);
|
||||
|
||||
var xz_stream = try xz.Decompress.init(&in_stream, gpa, &.{});
|
||||
defer xz_stream.deinit();
|
||||
|
||||
return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
|
||||
return xz_stream.reader.allocRemaining(gpa, .unlimited);
|
||||
}
|
||||
|
||||
fn testReader(data: []const u8, comptime expected: []const u8) !void {
|
||||
const buf = try decompress(data);
|
||||
defer testing.allocator.free(buf);
|
||||
fn testReader(data: []const u8, expected: []const u8) !void {
|
||||
const gpa = testing.allocator;
|
||||
|
||||
try testing.expectEqualSlices(u8, expected, buf);
|
||||
const result = try decompress(data);
|
||||
defer gpa.free(result);
|
||||
|
||||
try testing.expectEqualSlices(u8, expected, result);
|
||||
}
|
||||
|
||||
test "compressed data" {
|
||||
fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
|
||||
const gpa = std.testing.allocator;
|
||||
var stream: std.Io.Reader = .fixed(compressed);
|
||||
|
||||
var decompressor = try xz.Decompress.init(&stream, gpa, &.{});
|
||||
defer decompressor.deinit();
|
||||
|
||||
try std.testing.expectError(error.ReadFailed, decompressor.reader.allocRemaining(gpa, .unlimited));
|
||||
try std.testing.expectEqual(expected, decompressor.err orelse return error.TestFailed);
|
||||
}
|
||||
|
||||
test "fixture good-0-empty.xz" {
|
||||
try testReader(@embedFile("testdata/good-0-empty.xz"), "");
|
||||
}
|
||||
|
||||
inline for ([_][]const u8{
|
||||
"good-1-check-none.xz",
|
||||
"good-1-check-crc32.xz",
|
||||
"good-1-check-crc64.xz",
|
||||
"good-1-check-sha256.xz",
|
||||
"good-2-lzma2.xz",
|
||||
"good-1-block_header-1.xz",
|
||||
"good-1-block_header-2.xz",
|
||||
"good-1-block_header-3.xz",
|
||||
}) |filename| {
|
||||
try testReader(@embedFile("testdata/" ++ filename),
|
||||
const hello_world_text =
|
||||
\\Hello
|
||||
\\World!
|
||||
\\
|
||||
);
|
||||
}
|
||||
;
|
||||
|
||||
inline for ([_][]const u8{
|
||||
"good-1-lzma2-1.xz",
|
||||
"good-1-lzma2-2.xz",
|
||||
"good-1-lzma2-3.xz",
|
||||
"good-1-lzma2-4.xz",
|
||||
}) |filename| {
|
||||
try testReader(@embedFile("testdata/" ++ filename),
|
||||
test "fixture good-1-check-none.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-check-none.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-check-crc32.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-check-crc32.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-check-crc64.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-check-crc64.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-check-sha256.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-check-sha256.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-2-lzma2.xz" {
|
||||
try testReader(@embedFile("testdata/good-2-lzma2.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-block_header-1.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-block_header-1.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-block_header-2.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-block_header-2.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-block_header-3.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-block_header-3.xz"), hello_world_text);
|
||||
}
|
||||
|
||||
const lorem_ipsum_text =
|
||||
\\Lorem ipsum dolor sit amet, consectetur adipisicing
|
||||
\\elit, sed do eiusmod tempor incididunt ut
|
||||
\\labore et dolore magna aliqua. Ut enim
|
||||
|
|
@ -56,27 +87,54 @@ test "compressed data" {
|
|||
\\non proident, sunt in culpa qui officia
|
||||
\\deserunt mollit anim id est laborum.
|
||||
\\
|
||||
);
|
||||
}
|
||||
;
|
||||
|
||||
test "fixture good-1-lzma2-1.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-lzma2-1.xz"), lorem_ipsum_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-lzma2-2.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-lzma2-2.xz"), lorem_ipsum_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-lzma2-3.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-lzma2-3.xz"), lorem_ipsum_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-lzma2-4.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-lzma2-4.xz"), lorem_ipsum_text);
|
||||
}
|
||||
|
||||
test "fixture good-1-lzma2-5.xz" {
|
||||
try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
|
||||
}
|
||||
|
||||
test "unsupported" {
|
||||
inline for ([_][]const u8{
|
||||
"good-1-delta-lzma2.tiff.xz",
|
||||
"good-1-x86-lzma2.xz",
|
||||
"good-1-sparc-lzma2.xz",
|
||||
"good-1-arm64-lzma2-1.xz",
|
||||
"good-1-arm64-lzma2-2.xz",
|
||||
"good-1-3delta-lzma2.xz",
|
||||
"good-1-empty-bcj-lzma2.xz",
|
||||
}) |filename| {
|
||||
try testing.expectError(
|
||||
error.Unsupported,
|
||||
decompress(@embedFile("testdata/" ++ filename)),
|
||||
);
|
||||
}
|
||||
test "fixture good-1-delta-lzma2.tiff.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-delta-lzma2.tiff.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-x86-lzma2.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-x86-lzma2.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-sparc-lzma2.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-sparc-lzma2.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-arm64-lzma2-1.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-arm64-lzma2-1.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-arm64-lzma2-2.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-arm64-lzma2-2.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-3delta-lzma2.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-3delta-lzma2.xz"));
|
||||
}
|
||||
|
||||
test "fixture good-1-empty-bcj-lzma2.xz" {
|
||||
try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-empty-bcj-lzma2.xz"));
|
||||
}
|
||||
|
||||
fn testDontPanic(data: []const u8) !void {
|
||||
|
|
@ -91,6 +149,8 @@ test "size fields: integer overflow avoidance" {
|
|||
// These cases were found via fuzz testing and each previously caused
|
||||
// an integer overflow when decoding. We just want to ensure they no longer
|
||||
// cause a panic
|
||||
// TODO this not a sufficient way to test. tests should always check the result,
|
||||
// not merely ensure that the code does not crash.
|
||||
const header_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6z";
|
||||
try testDontPanic(header_size_overflow);
|
||||
const lzma2_chunk_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6\x02\x00!\x01\x08\x00\x00\x00\xd8\x0f#\x13\x01\xff\xff";
|
||||
|
|
|
|||
|
|
@ -1204,12 +1204,10 @@ fn unpackResource(
|
|||
},
|
||||
.@"tar.xz" => {
|
||||
const gpa = f.arena.child_allocator;
|
||||
var dcp = std.compress.xz.decompress(gpa, resource.reader().adaptToOldInterface()) catch |err|
|
||||
var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err|
|
||||
return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
|
||||
defer dcp.deinit();
|
||||
var adapter_buffer: [1024]u8 = undefined;
|
||||
var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
|
||||
return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
|
||||
defer decompress.deinit();
|
||||
return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
|
||||
},
|
||||
.@"tar.zst" => {
|
||||
const window_len = std.compress.zstd.default_window_len;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue