From d1c14f2f52ddec476eca6d605b985a27f4d4fe28 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 21:47:26 -0700 Subject: [PATCH] std.Build.Step.WriteFile: extract UpdateSourceFiles This has been planned for quite some time; this commit finally does it. Also implements file system watching integration in the make() implementation for UpdateSourceFiles and fixes the reporting of step caching for both. WriteFile does not yet have file system watching integration. --- build.zig | 2 +- lib/std/Build.zig | 4 + lib/std/Build/Step.zig | 4 + lib/std/Build/Step/UpdateSourceFiles.zig | 114 +++++++++++++++++++++++ lib/std/Build/Step/WriteFile.zig | 89 +----------------- test/tests.zig | 2 +- 6 files changed, 128 insertions(+), 87 deletions(-) create mode 100644 lib/std/Build/Step/UpdateSourceFiles.zig diff --git a/build.zig b/build.zig index 0f0d7d4d67..a364982ce9 100644 --- a/build.zig +++ b/build.zig @@ -595,7 +595,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void { run_opt.addArg("-o"); run_opt.addFileArg(b.path("stage1/zig1.wasm")); - const copy_zig_h = b.addWriteFiles(); + const copy_zig_h = b.addUpdateSourceFiles(); copy_zig_h.addCopyFileToSource(b.path("lib/zig.h"), "stage1/zig.h"); const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm"); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 72108adaf5..87bb0eeeda 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1052,6 +1052,10 @@ pub fn addWriteFiles(b: *Build) *Step.WriteFile { return Step.WriteFile.create(b); } +pub fn addUpdateSourceFiles(b: *Build) *Step.UpdateSourceFiles { + return Step.UpdateSourceFiles.create(b); +} + pub fn addRemoveDirTree(b: *Build, dir_path: LazyPath) *Step.RemoveDir { return Step.RemoveDir.create(b, dir_path); } diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 91d3924611..e41912d548 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -102,6 +102,7 @@ pub const Id = enum { fmt, translate_c, write_file, + update_source_files, run, check_file, check_object, @@ -122,6 +123,7 @@ pub const Id = enum { .fmt => Fmt, .translate_c => TranslateC, .write_file => WriteFile, + .update_source_files => UpdateSourceFiles, .run => Run, .check_file => CheckFile, .check_object => CheckObject, @@ -148,6 +150,7 @@ pub const RemoveDir = @import("Step/RemoveDir.zig"); pub const Run = @import("Step/Run.zig"); pub const TranslateC = @import("Step/TranslateC.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); +pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); pub const Inputs = struct { table: Table, @@ -680,4 +683,5 @@ test { _ = Run; _ = TranslateC; _ = WriteFile; + _ = UpdateSourceFiles; } diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig new file mode 100644 index 0000000000..9d1c8e20fe --- /dev/null +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -0,0 +1,114 @@ +//! Writes data to paths relative to the package root, effectively mutating the +//! package's source files. Be careful with the latter functionality; it should +//! not be used during the normal build process, but as a utility run by a +//! developer with intention to update source files, which will then be +//! committed to version control. +const std = @import("std"); +const Step = std.Build.Step; +const fs = std.fs; +const ArrayList = std.ArrayList; +const UpdateSourceFiles = @This(); + +step: Step, +output_source_files: std.ArrayListUnmanaged(OutputSourceFile), + +pub const base_id: Step.Id = .update_source_files; + +pub const OutputSourceFile = struct { + contents: Contents, + sub_path: []const u8, +}; + +pub const Contents = union(enum) { + bytes: []const u8, + copy: std.Build.LazyPath, +}; + +pub fn create(owner: *std.Build) *UpdateSourceFiles { + const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); + usf.* = .{ + .step = Step.init(.{ + .id = base_id, + .name = "UpdateSourceFiles", + .owner = owner, + .makeFn = make, + }), + .output_source_files = .{}, + }; + return usf; +} + +/// A path relative to the package root. +/// +/// Be careful with this because it updates source files. This should not be +/// used as part of the normal build process, but as a utility occasionally +/// run by a developer with intent to modify source files and then commit +/// those changes to version control. +pub fn addCopyFileToSource(usf: *UpdateSourceFiles, source: std.Build.LazyPath, sub_path: []const u8) void { + const b = usf.step.owner; + usf.output_source_files.append(b.allocator, .{ + .contents = .{ .copy = source }, + .sub_path = sub_path, + }) catch @panic("OOM"); + source.addStepDependencies(&usf.step); +} + +/// A path relative to the package root. +/// +/// Be careful with this because it updates source files. This should not be +/// used as part of the normal build process, but as a utility occasionally +/// run by a developer with intent to modify source files and then commit +/// those changes to version control. +pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []const u8) void { + const b = usf.step.owner; + usf.output_source_files.append(b.allocator, .{ + .contents = .{ .bytes = bytes }, + .sub_path = sub_path, + }) catch @panic("OOM"); +} + +fn make(step: *Step, prog_node: std.Progress.Node) !void { + _ = prog_node; + const b = step.owner; + const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); + + var any_miss = false; + for (usf.output_source_files.items) |output_source_file| { + if (fs.path.dirname(output_source_file.sub_path)) |dirname| { + b.build_root.handle.makePath(dirname) catch |err| { + return step.fail("unable to make path '{}{s}': {s}", .{ + b.build_root, dirname, @errorName(err), + }); + }; + } + switch (output_source_file.contents) { + .bytes => |bytes| { + b.build_root.handle.writeFile(.{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { + return step.fail("unable to write file '{}{s}': {s}", .{ + b.build_root, output_source_file.sub_path, @errorName(err), + }); + }; + any_miss = true; + }, + .copy => |file_source| { + if (!step.inputs.populated()) try step.addWatchInput(file_source); + + const source_path = file_source.getPath2(b, step); + const prev_status = fs.Dir.updateFile( + fs.cwd(), + source_path, + b.build_root.handle, + output_source_file.sub_path, + .{}, + ) catch |err| { + return step.fail("unable to update file from '{s}' to '{}{s}': {s}", .{ + source_path, b.build_root, output_source_file.sub_path, @errorName(err), + }); + }; + any_miss = any_miss or prev_status == .stale; + }, + } + } + + step.result_cached = !any_miss; +} diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 013c58890a..f35bf09b7e 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -1,13 +1,6 @@ -//! WriteFile is primarily used to create a directory in an appropriate -//! location inside the local cache which has a set of files that have either -//! been generated during the build, or are copied from the source package. -//! -//! However, this step has an additional capability of writing data to paths -//! relative to the package root, effectively mutating the package's source -//! files. Be careful with the latter functionality; it should not be used -//! during the normal build process, but as a utility run by a developer with -//! intention to update source files, which will then be committed to version -//! control. +//! WriteFile is used to create a directory in an appropriate location inside +//! the local cache which has a set of files that have either been generated +//! during the build, or are copied from the source package. const std = @import("std"); const Step = std.Build.Step; const fs = std.fs; @@ -19,8 +12,6 @@ step: Step, // The elements here are pointers because we need stable pointers for the GeneratedFile field. files: std.ArrayListUnmanaged(File), directories: std.ArrayListUnmanaged(Directory), - -output_source_files: std.ArrayListUnmanaged(OutputSourceFile), generated_directory: std.Build.GeneratedFile, pub const base_id: Step.Id = .write_file; @@ -52,11 +43,6 @@ pub const Directory = struct { }; }; -pub const OutputSourceFile = struct { - contents: Contents, - sub_path: []const u8, -}; - pub const Contents = union(enum) { bytes: []const u8, copy: std.Build.LazyPath, @@ -73,7 +59,6 @@ pub fn create(owner: *std.Build) *WriteFile { }), .files = .{}, .directories = .{}, - .output_source_files = .{}, .generated_directory = .{ .step = &write_file.step }, }; return write_file; @@ -150,33 +135,6 @@ pub fn addCopyDirectory( }; } -/// A path relative to the package root. -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addCopyFileToSource(write_file: *WriteFile, source: std.Build.LazyPath, sub_path: []const u8) void { - const b = write_file.step.owner; - write_file.output_source_files.append(b.allocator, .{ - .contents = .{ .copy = source }, - .sub_path = sub_path, - }) catch @panic("OOM"); - source.addStepDependencies(&write_file.step); -} - -/// A path relative to the package root. -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addBytesToSource(write_file: *WriteFile, bytes: []const u8, sub_path: []const u8) void { - const b = write_file.step.owner; - write_file.output_source_files.append(b.allocator, .{ - .contents = .{ .bytes = bytes }, - .sub_path = sub_path, - }) catch @panic("OOM"); -} - /// Returns a `LazyPath` representing the base directory that contains all the /// files from this `WriteFile`. pub fn getDirectory(write_file: *WriteFile) std.Build.LazyPath { @@ -202,46 +160,6 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const b = step.owner; const write_file: *WriteFile = @fieldParentPtr("step", step); - // Writing to source files is kind of an extra capability of this - // WriteFile - arguably it should be a different step. But anyway here - // it is, it happens unconditionally and does not interact with the other - // files here. - var any_miss = false; - for (write_file.output_source_files.items) |output_source_file| { - if (fs.path.dirname(output_source_file.sub_path)) |dirname| { - b.build_root.handle.makePath(dirname) catch |err| { - return step.fail("unable to make path '{}{s}': {s}", .{ - b.build_root, dirname, @errorName(err), - }); - }; - } - switch (output_source_file.contents) { - .bytes => |bytes| { - b.build_root.handle.writeFile(.{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{}{s}': {s}", .{ - b.build_root, output_source_file.sub_path, @errorName(err), - }); - }; - any_miss = true; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = fs.Dir.updateFile( - fs.cwd(), - source_path, - b.build_root.handle, - output_source_file.sub_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{s}' to '{}{s}': {s}", .{ - source_path, b.build_root, output_source_file.sub_path, @errorName(err), - }); - }; - any_miss = any_miss or prev_status == .stale; - }, - } - } - // The cache is used here not really as a way to speed things up - because writing // the data to a file would probably be very fast - but as a way to find a canonical // location to put build artifacts. @@ -278,6 +196,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { if (try step.cacheHit(&man)) { const digest = man.final(); write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest }); + step.result_cached = true; return; } diff --git a/test/tests.zig b/test/tests.zig index e19a9efccf..0862f8deb0 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -882,7 +882,7 @@ pub fn addCliTests(b: *std.Build) *Step { const unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00"; const fmt6_path = std.fs.path.join(b.allocator, &.{ tmp_path, "fmt6.zig" }) catch @panic("OOM"); - const write6 = b.addWriteFiles(); + const write6 = b.addUpdateSourceFiles(); write6.addBytesToSource(unformatted_code_utf16, fmt6_path); write6.step.dependOn(&run5.step);