mirror of
https://codeberg.org/ziglang/zig.git
synced 2025-12-06 13:54:21 +00:00
std.Build.WriteFileStep: integrate with cache system
And additionally support writing files to source files. This means a custom build step in zig's own build.zig is no longer needed for copying zig.h because it is handled by WriteFileStep.
This commit is contained in:
Andrew Kelley
parent
c9e02d3e69
commit
d3c9bfada6
2 changed files with 175 additions and 100 deletions
31
build.zig
31
build.zig
|
|
@ -509,35 +509,8 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
|
|||
run_opt.addArg("-o");
|
||||
run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
|
||||
|
||||
const CopyFileStep = struct {
|
||||
const Step = std.Build.Step;
|
||||
const FileSource = std.Build.FileSource;
|
||||
const CopyFileStep = @This();
|
||||
|
||||
step: Step,
|
||||
builder: *std.Build,
|
||||
source: FileSource,
|
||||
dest_rel_path: []const u8,
|
||||
|
||||
pub fn init(builder: *std.Build, source: FileSource, dest_rel_path: []const u8) CopyFileStep {
|
||||
return CopyFileStep{
|
||||
.builder = builder,
|
||||
.step = Step.init(.custom, builder.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), builder.allocator, make),
|
||||
.source = source.dupe(builder),
|
||||
.dest_rel_path = builder.dupePath(dest_rel_path),
|
||||
};
|
||||
}
|
||||
|
||||
fn make(step: *Step) !void {
|
||||
const self = @fieldParentPtr(CopyFileStep, "step", step);
|
||||
const full_src_path = self.source.getPath(self.builder);
|
||||
const full_dest_path = self.builder.pathFromRoot(self.dest_rel_path);
|
||||
try self.builder.updateFile(full_src_path, full_dest_path);
|
||||
}
|
||||
};
|
||||
|
||||
const copy_zig_h = try b.allocator.create(CopyFileStep);
|
||||
copy_zig_h.* = CopyFileStep.init(b, .{ .path = "lib/zig.h" }, "stage1/zig.h");
|
||||
const copy_zig_h = b.addWriteFiles();
|
||||
copy_zig_h.addCopyFileToSource(.{ .path = "lib/zig.h" }, "stage1/zig.h");
|
||||
|
||||
const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");
|
||||
update_zig1_step.dependOn(&run_opt.step);
|
||||
|
|
|
|||
|
|
@ -1,55 +1,117 @@
|
|||
const std = @import("../std.zig");
|
||||
const Step = std.Build.Step;
|
||||
const fs = std.fs;
|
||||
const ArrayList = std.ArrayList;
|
||||
|
||||
const WriteFileStep = @This();
|
||||
|
||||
pub const base_id = .write_file;
|
||||
//! WriteFileStep is primarily used to create a directory in an appropriate
|
||||
//! location inside the local cache which has a set of files that have either
|
||||
//! been generated during the build, or are copied from the source package.
|
||||
//!
|
||||
//! However, this step has an additional capability of writing data to paths
|
||||
//! relative to the package root, effectively mutating the package's source
|
||||
//! files. Be careful with the latter functionality; it should not be used
|
||||
//! during the normal build process, but as a utility run by a developer with
|
||||
//! intention to update source files, which will then be committed to version
|
||||
//! control.
|
||||
|
||||
step: Step,
|
||||
builder: *std.Build,
|
||||
files: std.TailQueue(File),
|
||||
/// The elements here are pointers because we need stable pointers for the
|
||||
/// GeneratedFile field.
|
||||
files: std.ArrayListUnmanaged(*File),
|
||||
output_source_files: std.ArrayListUnmanaged(OutputSourceFile),
|
||||
|
||||
pub const base_id = .write_file;
|
||||
|
||||
pub const File = struct {
|
||||
source: std.Build.GeneratedFile,
|
||||
basename: []const u8,
|
||||
generated_file: std.Build.GeneratedFile,
|
||||
sub_path: []const u8,
|
||||
contents: Contents,
|
||||
};
|
||||
|
||||
pub const OutputSourceFile = struct {
|
||||
contents: Contents,
|
||||
sub_path: []const u8,
|
||||
};
|
||||
|
||||
pub const Contents = union(enum) {
|
||||
bytes: []const u8,
|
||||
copy: std.Build.FileSource,
|
||||
};
|
||||
|
||||
pub fn init(builder: *std.Build) WriteFileStep {
|
||||
return WriteFileStep{
|
||||
return .{
|
||||
.builder = builder,
|
||||
.step = Step.init(.write_file, "writefile", builder.allocator, make),
|
||||
.files = .{},
|
||||
.output_source_files = .{},
|
||||
};
|
||||
}
|
||||
|
||||
pub fn add(self: *WriteFileStep, basename: []const u8, bytes: []const u8) void {
|
||||
const node = self.builder.allocator.create(std.TailQueue(File).Node) catch @panic("unhandled error");
|
||||
node.* = .{
|
||||
.data = .{
|
||||
.source = std.Build.GeneratedFile{ .step = &self.step },
|
||||
.basename = self.builder.dupePath(basename),
|
||||
.bytes = self.builder.dupe(bytes),
|
||||
},
|
||||
pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
|
||||
const gpa = wf.builder.allocator;
|
||||
const file = gpa.create(File) catch @panic("OOM");
|
||||
file.* = .{
|
||||
.generated_file = .{ .step = &wf.step },
|
||||
.sub_path = wf.builder.dupePath(sub_path),
|
||||
.contents = .{ .bytes = wf.builder.dupe(bytes) },
|
||||
};
|
||||
|
||||
self.files.append(node);
|
||||
wf.files.append(gpa, file) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// Gets a file source for the given basename. If the file does not exist, returns `null`.
|
||||
pub fn getFileSource(step: *WriteFileStep, basename: []const u8) ?std.Build.FileSource {
|
||||
var it = step.files.first;
|
||||
while (it) |node| : (it = node.next) {
|
||||
if (std.mem.eql(u8, node.data.basename, basename))
|
||||
return std.Build.FileSource{ .generated = &node.data.source };
|
||||
/// Place the file into the generated directory within the local cache,
|
||||
/// along with all the rest of the files added to this step. The parameter
|
||||
/// here is the destination path relative to the local cache directory
|
||||
/// associated with this WriteFileStep. It may be a basename, or it may
|
||||
/// include sub-directories, in which case this step will ensure the
|
||||
/// required sub-path exists.
|
||||
/// This is the option expected to be used most commonly with `addCopyFile`.
|
||||
pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
|
||||
const gpa = wf.builder.allocator;
|
||||
const file = gpa.create(File) catch @panic("OOM");
|
||||
file.* = .{
|
||||
.generated_file = .{ .step = &wf.step },
|
||||
.sub_path = wf.builder.dupePath(sub_path),
|
||||
.contents = .{ .copy = source },
|
||||
};
|
||||
wf.files.append(gpa, file) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// A path relative to the package root.
|
||||
/// Be careful with this because it updates source files. This should not be
|
||||
/// used as part of the normal build process, but as a utility occasionally
|
||||
/// run by a developer with intent to modify source files and then commit
|
||||
/// those changes to version control.
|
||||
/// A file added this way is not available with `getFileSource`.
|
||||
pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
|
||||
wf.output_source_files.append(wf.builder.allocator, .{
|
||||
.contents = .{ .copy = source },
|
||||
.sub_path = sub_path,
|
||||
}) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// Gets a file source for the given sub_path. If the file does not exist, returns `null`.
|
||||
pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSource {
|
||||
for (wf.files.items) |file| {
|
||||
if (std.mem.eql(u8, file.sub_path, sub_path)) {
|
||||
return .{ .generated = &file.generated_file };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn make(step: *Step) !void {
|
||||
const self = @fieldParentPtr(WriteFileStep, "step", step);
|
||||
const wf = @fieldParentPtr(WriteFileStep, "step", step);
|
||||
|
||||
// Writing to source files is kind of an extra capability of this
|
||||
// WriteFileStep - arguably it should be a different step. But anyway here
|
||||
// it is, it happens unconditionally and does not interact with the other
|
||||
// files here.
|
||||
for (wf.output_source_files.items) |output_source_file| {
|
||||
const basename = fs.path.basename(output_source_file.sub_path);
|
||||
if (fs.path.dirname(output_source_file.sub_path)) |dirname| {
|
||||
var dir = try wf.builder.build_root.handle.makeOpenPath(dirname, .{});
|
||||
defer dir.close();
|
||||
try writeFile(wf, dir, output_source_file.contents, basename);
|
||||
} else {
|
||||
try writeFile(wf, wf.builder.build_root.handle, output_source_file.contents, basename);
|
||||
}
|
||||
}
|
||||
|
||||
// The cache is used here not really as a way to speed things up - because writing
|
||||
// the data to a file would probably be very fast - but as a way to find a canonical
|
||||
|
|
@ -58,56 +120,96 @@ fn make(step: *Step) !void {
|
|||
// If, for example, a hard-coded path was used as the location to put WriteFileStep
|
||||
// files, then two WriteFileSteps executing in parallel might clobber each other.
|
||||
|
||||
// TODO port the cache system from the compiler to zig std lib. Until then
|
||||
// we directly construct the path, and no "cache hit" detection happens;
|
||||
// the files are always written.
|
||||
// Note there is similar code over in ConfigHeaderStep.
|
||||
const Hasher = std.crypto.auth.siphash.SipHash128(1, 3);
|
||||
var man = wf.builder.cache.obtain();
|
||||
defer man.deinit();
|
||||
|
||||
// Random bytes to make WriteFileStep unique. Refresh this with
|
||||
// new random bytes when WriteFileStep implementation is modified
|
||||
// in a non-backwards-compatible way.
|
||||
var hash = Hasher.init("eagVR1dYXoE7ARDP");
|
||||
man.hash.add(@as(u32, 0xd767ee59));
|
||||
|
||||
{
|
||||
var it = self.files.first;
|
||||
while (it) |node| : (it = node.next) {
|
||||
hash.update(node.data.basename);
|
||||
hash.update(node.data.bytes);
|
||||
hash.update("|");
|
||||
for (wf.files.items) |file| {
|
||||
man.hash.addBytes(file.sub_path);
|
||||
switch (file.contents) {
|
||||
.bytes => |bytes| {
|
||||
man.hash.addBytes(bytes);
|
||||
},
|
||||
.copy => |file_source| {
|
||||
_ = try man.addFile(file_source.getPath(wf.builder), null);
|
||||
},
|
||||
}
|
||||
}
|
||||
var digest: [16]u8 = undefined;
|
||||
hash.final(&digest);
|
||||
var hash_basename: [digest.len * 2]u8 = undefined;
|
||||
_ = std.fmt.bufPrint(
|
||||
&hash_basename,
|
||||
"{s}",
|
||||
.{std.fmt.fmtSliceHexLower(&digest)},
|
||||
) catch unreachable;
|
||||
|
||||
const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
|
||||
"o", &hash_basename,
|
||||
});
|
||||
var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
|
||||
std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
|
||||
return err;
|
||||
};
|
||||
defer dir.close();
|
||||
{
|
||||
var it = self.files.first;
|
||||
while (it) |node| : (it = node.next) {
|
||||
dir.writeFile(node.data.basename, node.data.bytes) catch |err| {
|
||||
std.debug.print("unable to write {s} into {s}: {s}\n", .{
|
||||
node.data.basename,
|
||||
output_dir,
|
||||
@errorName(err),
|
||||
});
|
||||
return err;
|
||||
};
|
||||
node.data.source.path = try fs.path.join(
|
||||
self.builder.allocator,
|
||||
&[_][]const u8{ output_dir, node.data.basename },
|
||||
if (man.hit() catch |err| failWithCacheError(man, err)) {
|
||||
// Cache hit, skip writing file data.
|
||||
const digest = man.final();
|
||||
for (wf.files.items) |file| {
|
||||
file.generated_file.path = try wf.builder.cache_root.join(
|
||||
wf.builder.allocator,
|
||||
&.{ "o", &digest, file.sub_path },
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const digest = man.final();
|
||||
const cache_path = "o" ++ fs.path.sep_str ++ digest;
|
||||
|
||||
var cache_dir = wf.builder.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
|
||||
std.debug.print("unable to make path {s}: {s}\n", .{ cache_path, @errorName(err) });
|
||||
return err;
|
||||
};
|
||||
defer cache_dir.close();
|
||||
|
||||
for (wf.files.items) |file| {
|
||||
const basename = fs.path.basename(file.sub_path);
|
||||
if (fs.path.dirname(file.sub_path)) |dirname| {
|
||||
var dir = try wf.builder.cache_root.handle.makeOpenPath(dirname, .{});
|
||||
defer dir.close();
|
||||
try writeFile(wf, dir, file.contents, basename);
|
||||
} else {
|
||||
try writeFile(wf, cache_dir, file.contents, basename);
|
||||
}
|
||||
|
||||
file.generated_file.path = try wf.builder.cache_root.join(
|
||||
wf.builder.allocator,
|
||||
&.{ cache_path, file.sub_path },
|
||||
);
|
||||
}
|
||||
|
||||
try man.writeManifest();
|
||||
}
|
||||
|
||||
fn writeFile(wf: *WriteFileStep, dir: fs.Dir, contents: Contents, basename: []const u8) !void {
|
||||
// TODO after landing concurrency PR, improve error reporting here
|
||||
switch (contents) {
|
||||
.bytes => |bytes| return dir.writeFile(basename, bytes),
|
||||
.copy => |file_source| {
|
||||
const source_path = file_source.getPath(wf.builder);
|
||||
const prev_status = try fs.Dir.updateFile(fs.cwd(), source_path, dir, basename, .{});
|
||||
_ = prev_status; // TODO logging (affected by open PR regarding concurrency)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// TODO consolidate this with the same function in RunStep?
|
||||
/// Also properly deal with concurrency (see open PR)
|
||||
fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
|
||||
const i = man.failed_file_index orelse failWithSimpleError(err);
|
||||
const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
|
||||
const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
|
||||
std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
|
||||
std.process.exit(1);
|
||||
}
|
||||
|
||||
fn failWithSimpleError(err: anyerror) noreturn {
|
||||
std.debug.print("{s}\n", .{@errorName(err)});
|
||||
std.process.exit(1);
|
||||
}
|
||||
|
||||
const std = @import("../std.zig");
|
||||
const Step = std.Build.Step;
|
||||
const fs = std.fs;
|
||||
const ArrayList = std.ArrayList;
|
||||
|
||||
const WriteFileStep = @This();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue