stage2: implement zig build

As part of this:

 * add std.process.cleanExit. closes #6395
   - use it in several places
 * adjust the alignment of text in `zig build --help` menu
 * Cache: support the concept of "unhit" so that we properly keep track
   of the cache when we find out using the secondary hash that the cache
   "hit" was actually a miss. Use this to fix false negatives of caching
   of stage1 build artifacts.
 * fix not deleting the symlink hash for stage1 build artifacts causing
   false positives.
 * implement support for Package arguments in stage1 build artifacts
 * update and add missing usage text
 * add --override-lib-dir and --enable-cache CLI options
   - `--enable-cache` takes the place of `--cache on`
 * CLI supports -femit-bin=foo combined with --enable-cache to do an
   "update file" operation. --enable-cache without that argument
   will build the output into a cache directory and then print the path
   to stdout (matching master branch behavior).
 * errors surfacing from main() now print "error: Foo" instead of
   "error: error.Foo".

Author: andrewrk

BRANCH_TODO

@@ -1,6 +1,3 @@
- * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
-   (maybe make it an explicit option and have main.zig disable it)
- * `zig build`
  * repair @cImport
  * make sure zig cc works
    - using it as a preprocessor (-E)
@@ -22,13 +19,16 @@
  * COFF LLD linking
  * WASM LLD linking
  * --main-pkg-path
+ * --pkg-begin, --pkg-end
+ * skip LLD caching when bin directory is not in the cache (so we don't put `id.txt` into the cwd)
+   (maybe make it an explicit option and have main.zig disable it)
  * audit the CLI options for stage2
  * audit the base cache hash
  * implement proper parsing of LLD stderr/stdout and exposing compile errors
  * implement proper parsing of clang stderr/stdout and exposing compile errors
  * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
  * restore error messages for stage2_add_link_lib
- * update zig build to use new CLI
+ * update std/build.zig to use new CLI
 
  * support cross compiling stage2 with `zig build`
  * implement proper compile errors for failing to build glibc crt files and shared libs

lib/std/build.zig

@@ -2294,8 +2294,7 @@ pub const LibExeObjStep = struct {
         if (self.kind == Kind.Test) {
             try builder.spawnChild(zig_args.span());
         } else {
-            try zig_args.append("--cache");
-            try zig_args.append("on");
+            try zig_args.append("--enable-cache");
 
             const output_dir_nl = try builder.execFromStep(zig_args.span(), &self.step);
             const build_output_dir = mem.trimRight(u8, output_dir_nl, "\r\n");

lib/std/process.zig

@@ -19,6 +19,19 @@ pub const exit = os.exit;
 pub const changeCurDir = os.chdir;
 pub const changeCurDirC = os.chdirC;
 
+/// Indicate that we are now terminating with a successful exit code.
+/// In debug builds, this is a no-op, so that the calling code's
+/// cleanup mechanisms are tested and so that external tools that
+/// check for resource leaks can be accurate. In release builds, this
+/// calls exit(0), and does not return.
+pub fn cleanExit() void {
+    if (builtin.mode == .Debug) {
+        return;
+    } else {
+        exit(0);
+    }
+}
+
 /// The result is a slice of `out_buffer`, from index `0`.
 pub fn getCwd(out_buffer: []u8) ![]u8 {
     return os.getcwd(out_buffer);

lib/std/special/build_runner.zig

@@ -161,16 +161,16 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void
             try fmt.allocPrint(allocator, "{} (default)", .{top_level_step.step.name})
         else
             top_level_step.step.name;
-        try out_stream.print("  {s:22} {}\n", .{ name, top_level_step.description });
+        try out_stream.print("  {s:<27} {}\n", .{ name, top_level_step.description });
     }
 
     try out_stream.writeAll(
         \\
         \\General Options:
-        \\  --help                 Print this help and exit
-        \\  --verbose              Print commands before executing them
-        \\  --prefix [path]        Override default install prefix
-        \\  --search-prefix [path] Add a path to look for binaries, libraries, headers
+        \\  --help                      Print this help and exit
+        \\  --verbose                   Print commands before executing them
+        \\  --prefix [path]             Override default install prefix
+        \\  --search-prefix [path]      Add a path to look for binaries, libraries, headers
         \\
         \\Project-Specific Options:
         \\
@@ -185,7 +185,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void
                 Builder.typeIdName(option.type_id),
             });
             defer allocator.free(name);
-            try out_stream.print("{s:32} {}\n", .{ name, option.description });
+            try out_stream.print("{s:<29} {}\n", .{ name, option.description });
         }
     }
 

src/Cache.zig

@@ -120,6 +120,13 @@ pub const HashHelper = struct {
         return copy.final();
     }
 
+    pub fn peekBin(hh: HashHelper) [bin_digest_len]u8 {
+        var copy = hh;
+        var bin_digest: [bin_digest_len]u8 = undefined;
+        copy.hasher.final(&bin_digest);
+        return bin_digest;
+    }
+
     /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
     pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
         var bin_digest: [bin_digest_len]u8 = undefined;
@@ -338,19 +345,7 @@ pub const CacheHash = struct {
         if (any_file_changed) {
             // cache miss
             // keep the manifest file open
-            // reset the hash
-            self.hash.hasher = hasher_init;
-            self.hash.hasher.update(&bin_digest);
-
-            // Remove files not in the initial hash
-            for (self.files.items[input_file_count..]) |*file| {
-                file.deinit(self.cache.gpa);
-            }
-            self.files.shrinkRetainingCapacity(input_file_count);
-
-            for (self.files.items) |file| {
-                self.hash.hasher.update(&file.bin_digest);
-            }
+            self.unhit(bin_digest, input_file_count);
             return false;
         }
 
@@ -366,6 +361,22 @@ pub const CacheHash = struct {
         return true;
     }
 
+    pub fn unhit(self: *CacheHash, bin_digest: [bin_digest_len]u8, input_file_count: usize) void {
+        // Reset the hash.
+        self.hash.hasher = hasher_init;
+        self.hash.hasher.update(&bin_digest);
+
+        // Remove files not in the initial hash.
+        for (self.files.items[input_file_count..]) |*file| {
+            file.deinit(self.cache.gpa);
+        }
+        self.files.shrinkRetainingCapacity(input_file_count);
+
+        for (self.files.items) |file| {
+            self.hash.hasher.update(&file.bin_digest);
+        }
+    }
+
     fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
         const file = try fs.cwd().openFile(ch_file.path.?, .{});
         defer file.close();

src/Compilation.zig

@@ -2200,20 +2200,34 @@ fn updateStage1Module(comp: *Compilation) !void {
     ch.hash.add(comp.bin_file.options.function_sections);
     ch.hash.add(comp.is_test);
 
+    // Capture the state in case we come back from this branch where the hash doesn't match.
+    const prev_hash_state = ch.hash.peekBin();
+    const input_file_count = ch.files.items.len;
+
     if (try ch.hit()) {
         const digest = ch.final();
 
         var prev_digest_buf: [digest.len]u8 = undefined;
         const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
+            log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
             // Handle this as a cache miss.
             break :blk prev_digest_buf[0..0];
         };
         if (mem.eql(u8, prev_digest, &digest)) {
+            log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
             comp.stage1_lock = ch.toOwnedLock();
             return;
         }
+        log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
+        ch.unhit(prev_hash_state, input_file_count);
     }
 
+    // We are about to change the output file to be different, so we invalidate the build hash now.
+    directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
+        error.FileNotFound => {},
+        else => |e| return e,
+    };
+
     const stage2_target = try arena.create(stage1.Stage2Target);
     stage2_target.* = .{
         .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
@@ -2243,16 +2257,7 @@ fn updateStage1Module(comp: *Compilation) !void {
         comp.is_test,
     ) orelse return error.OutOfMemory;
 
-    const stage1_pkg = try arena.create(stage1.Pkg);
-    stage1_pkg.* = .{
-        .name_ptr = undefined,
-        .name_len = 0,
-        .path_ptr = undefined,
-        .path_len = 0,
-        .children_ptr = undefined,
-        .children_len = 0,
-        .parent = null,
-    };
+    const stage1_pkg = try createStage1Pkg(arena, "root", mod.root_pkg, null);
     const output_dir = comp.bin_file.options.directory.path orelse ".";
     const test_filter = comp.test_filter orelse ""[0..0];
     const test_name_prefix = comp.test_name_prefix orelse ""[0..0];
@@ -2303,10 +2308,12 @@ fn updateStage1Module(comp: *Compilation) !void {
 
     const digest = ch.final();
 
+    log.debug("stage1 {} final digest={}", .{ mod.root_pkg.root_src_path, digest });
+
     // Update the dangling symlink with the digest. If it fails we can continue; it only
     // means that the next invocation will have an unnecessary cache miss.
     directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
-        std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+        std.log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)});
     };
     // Again failure here only means an unnecessary cache miss.
     ch.writeManifest() catch |err| {
@@ -2316,3 +2323,34 @@ fn updateStage1Module(comp: *Compilation) !void {
     // other processes clobbering it.
     comp.stage1_lock = ch.toOwnedLock();
 }
+
+fn createStage1Pkg(
+    arena: *Allocator,
+    name: []const u8,
+    pkg: *Package,
+    parent_pkg: ?*stage1.Pkg,
+) error{OutOfMemory}!*stage1.Pkg {
+    const child_pkg = try arena.create(stage1.Pkg);
+
+    const pkg_children = blk: {
+        var children = std.ArrayList(*stage1.Pkg).init(arena);
+        var it = pkg.table.iterator();
+        while (it.next()) |entry| {
+            try children.append(try createStage1Pkg(arena, entry.key, entry.value, child_pkg));
+        }
+        break :blk children.items;
+    };
+
+    const src_path = try pkg.root_src_directory.join(arena, &[_][]const u8{pkg.root_src_path});
+
+    child_pkg.* = .{
+        .name_ptr = name.ptr,
+        .name_len = name.len,
+        .path_ptr = src_path.ptr,
+        .path_len = src_path.len,
+        .children_ptr = pkg_children.ptr,
+        .children_len = pkg_children.len,
+        .parent = parent_pkg,
+    };
+    return child_pkg;
+}