make zig compiler processes live across rebuilds

Changes the `make` function signature to take an options struct, which
additionally includes `watch: bool`. I intentionally am not exposing
this information to configure phase logic.

Also adds global zig cache to the compiler cache prefixes.

Closes ziglang#20600

lib/compiler/build_runner.zig

@@ -1031,7 +1031,11 @@ fn workerMakeOneStep(
     const sub_prog_node = prog_node.start(s.name, 0);
     defer sub_prog_node.end();
 
-    const make_result = s.make(sub_prog_node);
+    const make_result = s.make(.{
+        .progress_node = sub_prog_node,
+        .thread_pool = thread_pool,
+        .watch = run.watch,
+    });
 
     // No matter the result, we want to display error/warning messages.
     const show_compile_errors = !run.prominent_compile_errors and

lib/std/Build.zig

@@ -1078,8 +1078,8 @@ pub fn getUninstallStep(b: *Build) *Step {
     return &b.uninstall_tls.step;
 }
 
-fn makeUninstall(uninstall_step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void {
+    _ = options;
     const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step);
     const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls);
 

lib/std/Build/Step.zig

@@ -68,7 +68,13 @@ pub const TestResults = struct {
     }
 };
 
-pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerror!void;
+pub const MakeOptions = struct {
+    progress_node: std.Progress.Node,
+    thread_pool: *std.Thread.Pool,
+    watch: bool,
+};
+
+pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
 
 pub const State = enum {
     precheck_unstarted,
@@ -219,10 +225,10 @@ pub fn init(options: StepOptions) Step {
 /// If the Step's `make` function reports `error.MakeFailed`, it indicates they
 /// have already reported the error. Otherwise, we add a simple error report
 /// here.
-pub fn make(s: *Step, prog_node: std.Progress.Node) error{ MakeFailed, MakeSkipped }!void {
+pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
     const arena = s.owner.allocator;
 
-    s.makeFn(s, prog_node) catch |err| switch (err) {
+    s.makeFn(s, options) catch |err| switch (err) {
         error.MakeFailed => return error.MakeFailed,
         error.MakeSkipped => return error.MakeSkipped,
         else => {
@@ -260,8 +266,8 @@ pub fn getStackTrace(s: *Step) ?std.builtin.StackTrace {
     };
 }
 
-fn makeNoOp(step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void {
+    _ = options;
 
     var all_cached = true;
 
@@ -352,13 +358,25 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO
     try step.result_error_msgs.append(arena, msg);
 }
 
+pub const ZigProcess = struct {
+    child: std.process.Child,
+    poller: std.io.Poller(StreamEnum),
+
+    pub const StreamEnum = enum { stdout, stderr };
+};
+
 /// Assumes that argv contains `--listen=-` and that the process being spawned
 /// is the zig compiler - the same version that compiled the build runner.
 pub fn evalZigProcess(
     s: *Step,
     argv: []const []const u8,
     prog_node: std.Progress.Node,
+    watch: bool,
 ) !?[]const u8 {
+    if (s.getZigProcess()) |zp| {
+        assert(watch);
+        return zigProcessUpdate(s, zp, watch);
+    }
     assert(argv.len != 0);
     const b = s.owner;
     const arena = b.allocator;
@@ -378,29 +396,76 @@ pub fn evalZigProcess(
     child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{
         argv[0], @errorName(err),
     });
-    var timer = try std.time.Timer.start();
 
-    var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
-        .stdout = child.stdout.?,
-        .stderr = child.stderr.?,
-    });
-    defer poller.deinit();
+    const zp = try arena.create(ZigProcess);
+    zp.* = .{
+        .child = child,
+        .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{
+            .stdout = child.stdout.?,
+            .stderr = child.stderr.?,
+        }),
+    };
+    if (watch) s.setZigProcess(zp);
+    defer if (!watch) zp.poller.deinit();
+
+    const result = try zigProcessUpdate(s, zp, watch);
+
+    if (!watch) {
+        // Send EOF to stdin.
+        zp.child.stdin.?.close();
+        zp.child.stdin = null;
+
+        const term = zp.child.wait() catch |err| {
+            return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
+        };
+        s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
+
+        // Special handling for Compile step that is expecting compile errors.
+        if (s.cast(Compile)) |compile| switch (term) {
+            .Exited => {
+                // Note that the exit code may be 0 in this case due to the
+                // compiler server protocol.
+                if (compile.expect_errors != null) {
+                    return error.NeedCompileErrorCheck;
+                }
+            },
+            else => {},
+        };
+
+        try handleChildProcessTerm(s, term, null, argv);
+    }
+
+    if (s.result_error_bundle.errorMessageCount() > 0) {
+        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
+            s.result_error_bundle.errorMessageCount(),
+            try allocPrintCmd(arena, null, argv),
+        });
+    }
+
+    return result;
+}
 
-    try sendMessage(child.stdin.?, .update);
-    try sendMessage(child.stdin.?, .exit);
+fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
+    const b = s.owner;
+    const arena = b.allocator;
+
+    var timer = try std.time.Timer.start();
+
+    try sendMessage(zp.child.stdin.?, .update);
+    if (!watch) try sendMessage(zp.child.stdin.?, .exit);
 
     const Header = std.zig.Server.Message.Header;
     var result: ?[]const u8 = null;
 
-    const stdout = poller.fifo(.stdout);
+    const stdout = zp.poller.fifo(.stdout);
 
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const header = stdout.reader().readStruct(Header) catch unreachable;
         while (stdout.readableLength() < header.bytes_len) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const body = stdout.readableSliceOfLen(header.bytes_len);
 
@@ -428,12 +493,22 @@ pub fn evalZigProcess(
                     .string_bytes = try arena.dupe(u8, string_bytes),
                     .extra = extra_array,
                 };
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .emit_bin_path => {
                 const EbpHdr = std.zig.Server.Message.EmitBinPath;
                 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
                 s.result_cached = ebp_hdr.flags.cache_hit;
                 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .file_system_inputs => {
                 s.clearWatchInputs();
@@ -470,6 +545,13 @@ pub fn evalZigProcess(
                             };
                             try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
                         },
+                        .global_cache => {
+                            const path: Build.Cache.Path = .{
+                                .root_dir = s.owner.graph.global_cache_root,
+                                .sub_path = sub_path_dirname,
+                            };
+                            try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
+                        },
                     }
                 }
             },
@@ -479,43 +561,28 @@ pub fn evalZigProcess(
         stdout.discard(body.len);
     }
 
-    const stderr = poller.fifo(.stderr);
+    s.result_duration_ns = timer.read();
+
+    const stderr = zp.poller.fifo(.stderr);
     if (stderr.readableLength() > 0) {
         try s.result_error_msgs.append(arena, try stderr.toOwnedSlice());
     }
 
-    // Send EOF to stdin.
-    child.stdin.?.close();
-    child.stdin = null;
+    return result;
+}
 
-    const term = child.wait() catch |err| {
-        return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
+fn getZigProcess(s: *Step) ?*ZigProcess {
+    return switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process,
+        else => null,
     };
-    s.result_duration_ns = timer.read();
-    s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
-
-    // Special handling for Compile step that is expecting compile errors.
-    if (s.cast(Compile)) |compile| switch (term) {
-        .Exited => {
-            // Note that the exit code may be 0 in this case due to the
-            // compiler server protocol.
-            if (compile.expect_errors != null) {
-                return error.NeedCompileErrorCheck;
-            }
-        },
-        else => {},
-    };
-
-    try handleChildProcessTerm(s, term, null, argv);
+}
 
-    if (s.result_error_bundle.errorMessageCount() > 0) {
-        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
-            s.result_error_bundle.errorMessageCount(),
-            try allocPrintCmd(arena, null, argv),
-        });
+fn setZigProcess(s: *Step, zp: *ZigProcess) void {
+    switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process = zp,
+        else => unreachable,
     }
-
-    return result;
 }
 
 fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {

lib/std/Build/Step/CheckFile.zig

@@ -46,8 +46,8 @@ pub fn setName(check_file: *CheckFile, name: []const u8) void {
     check_file.step.name = name;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const check_file: *CheckFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(check_file.source);

lib/std/Build/Step/CheckObject.zig

@@ -550,8 +550,8 @@ pub fn checkComputeCompare(
     check_object.checks.append(check) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, make_options: Step.MakeOptions) !void {
+    _ = make_options;
     const b = step.owner;
     const gpa = b.allocator;
     const check_object: *CheckObject = @fieldParentPtr("step", step);

lib/std/Build/Step/Compile.zig

@@ -213,6 +213,10 @@ is_linking_libcpp: bool = false,
 
 no_builtin: bool = false,
 
+/// Populated during the make phase when there is a long-lived compiler process.
+/// Managed by the build runner, not user build script.
+zig_process: ?*Step.ZigProcess,
+
 pub const ExpectedCompileErrors = union(enum) {
     contains: []const u8,
     exact: []const []const u8,
@@ -398,6 +402,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
 
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
+
+        .zig_process = null,
     };
 
     compile.root_module.init(owner, options.root_module, compile);
@@ -1735,13 +1741,17 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
     return try zig_args.toOwnedSlice();
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
     const b = step.owner;
     const compile: *Compile = @fieldParentPtr("step", step);
 
     const zig_args = try getZigArgs(compile);
 
-    const maybe_output_bin_path = step.evalZigProcess(zig_args, prog_node) catch |err| switch (err) {
+    const maybe_output_bin_path = step.evalZigProcess(
+        zig_args,
+        options.progress_node,
+        options.watch,
+    ) catch |err| switch (err) {
         error.NeedCompileErrorCheck => {
             assert(compile.expect_errors != null);
             try checkCompileErrors(compile);

lib/std/Build/Step/ConfigHeader.zig

@@ -164,8 +164,8 @@ fn putValue(config_header: *ConfigHeader, field_name: []const u8, comptime T: ty
     }
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const config_header: *ConfigHeader = @fieldParentPtr("step", step);
     if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp);

lib/std/Build/Step/Fail.zig

@@ -24,8 +24,8 @@ pub fn create(owner: *std.Build, error_msg: []const u8) *Fail {
     return fail;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node; // No progress to report.
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options; // No progress to report.
 
     const fail: *Fail = @fieldParentPtr("step", step);
 

lib/std/Build/Step/Fmt.zig

@@ -36,7 +36,9 @@ pub fn create(owner: *std.Build, options: Options) *Fmt {
     return fmt;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
+
     // TODO: if check=false, this means we are modifying source files in place, which
     // is an operation that could race against other operations also modifying source files
     // in place. In this case, this step should obtain a write lock while making those

lib/std/Build/Step/InstallArtifact.zig

@@ -115,8 +115,8 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins
     return install_artifact;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const install_artifact: *InstallArtifact = @fieldParentPtr("step", step);
     const b = step.owner;
     const cwd = fs.cwd();

lib/std/Build/Step/InstallDir.zig

@@ -55,8 +55,8 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir {
     return install_dir;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_dir: *InstallDir = @fieldParentPtr("step", step);
     step.clearWatchInputs();

lib/std/Build/Step/InstallFile.zig

@@ -35,8 +35,8 @@ pub fn create(
     return install_file;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_file: *InstallFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(install_file.source);