move installation logic to the build script where it belongs

* build.zig: introduce `-Dflat` option which makes the installation
  match what we want to ship for our download tarballs. This allows
  deleting a bunch of shell script logic from the CI.
  - for example it puts the executable directly in prefix/zig rather
    than prefix/bin/zig and it additionally includes prefix/LICENSE.
* build.zig: by default also install std lib documentation to doc/std/
  - this can be disabled by `-Dno-autodocs` similar to how there is
    already `-Dno-langref`.
* build.zig: add `std-docs` and `langref` steps which build and install
  the std lib autodocs and langref to prefix/doc/std and
  prefix/doc/langref.html, respectively.

* std.Build: implement proper handling of `-femit-docs` using the
  LazyPath system. This is a breaking change.
  - this is a partial implementation of #16351
* frontend: fixed the handling of Autodocs with regards to caching and
  putting the artifacts in the proper location to integrate with the
  build system.
  - closes #15864

* CI: delete the logic for autodocs since it is now handled by build.zig
  and is enabled by default.
  - in the future we should strive to have nearly all the CI shell
    script logic deleted in favor of `zig build` commands.
* CI: pass `-DZIG_NO_LIB=ON`/`-Dno-lib` except for the one command where
  we want to actually generate the langref and autodocs. Generating the
  langref takes 14 minutes right now (why?!) so we don't want to do that
  more times than necessary.

* Autodoc: fixed use of a global variable. It works fine as a local
  variable instead.
  - note that in the future we will want to make Autodoc run
    simultaneously using the job system, but for now the principle of
    YAGNI dictates that we don't have an init()/deinit() API and instead
    simply call the function that does the things.
* Autodoc: only do it when there are no compile errors

build.zig

@@ -24,12 +24,14 @@ pub fn build(b: *std.Build) !void {
 
     const optimize = b.standardOptimizeOption(.{});
 
+    const flat = b.option(bool, "flat", "Put files into the installation prefix in a manner suited for upstream distribution rather than a posix file system hierarchy standard") orelse false;
     const single_threaded = b.option(bool, "single-threaded", "Build artifacts that run in single threaded mode");
     const use_zig_libcxx = b.option(bool, "use-zig-libcxx", "If libc++ is needed, use zig's bundled version, don't try to integrate with the system") orelse false;
 
     const test_step = b.step("test", "Run all the tests");
     const skip_install_lib_files = b.option(bool, "no-lib", "skip copying of lib/ files and langref to installation prefix. Useful for development") orelse false;
     const skip_install_langref = b.option(bool, "no-langref", "skip copying of langref to the installation prefix") orelse skip_install_lib_files;
+    const skip_install_autodocs = b.option(bool, "no-autodocs", "skip copying of standard library autodocs to the installation prefix") orelse skip_install_lib_files;
     const no_bin = b.option(bool, "no-bin", "skip emitting compiler binary") orelse false;
 
     const docgen_exe = b.addExecutable(.{
@@ -52,8 +54,34 @@ pub fn build(b: *std.Build) !void {
         b.getInstallStep().dependOn(&install_langref.step);
     }
 
-    const docs_step = b.step("docs", "Build documentation");
-    docs_step.dependOn(&docgen_cmd.step);
+    const autodoc_test = b.addTest(.{
+        .root_source_file = .{ .path = "lib/std/std.zig" },
+        .target = target,
+    });
+    autodoc_test.overrideZigLibDir("lib");
+    autodoc_test.emit_bin = .no_emit; // https://github.com/ziglang/zig/issues/16351
+    const install_std_docs = b.addInstallDirectory(.{
+        .source_dir = autodoc_test.getOutputDocs(),
+        .install_dir = .prefix,
+        .install_subdir = "doc/std",
+    });
+    if (!skip_install_autodocs) {
+        b.getInstallStep().dependOn(&install_std_docs.step);
+    }
+
+    if (flat) {
+        b.installFile("LICENSE", "LICENSE");
+    }
+
+    const langref_step = b.step("langref", "Build and install the language reference");
+    langref_step.dependOn(&install_langref.step);
+
+    const std_docs_step = b.step("std-docs", "Build and install the standard library documentation");
+    std_docs_step.dependOn(&install_std_docs.step);
+
+    const docs_step = b.step("docs", "Build and install documentation");
+    docs_step.dependOn(langref_step);
+    docs_step.dependOn(std_docs_step);
 
     const check_case_exe = b.addExecutable(.{
         .name = "check-case",
@@ -104,10 +132,10 @@ pub fn build(b: *std.Build) !void {
     const config_h_path_option = b.option([]const u8, "config_h", "Path to the generated config.h");
 
     if (!skip_install_lib_files) {
-        b.installDirectory(InstallDirectoryOptions{
+        b.installDirectory(.{
             .source_dir = .{ .path = "lib" },
-            .install_dir = .lib,
-            .install_subdir = "zig",
+            .install_dir = if (flat) .prefix else .lib,
+            .install_subdir = if (flat) "lib" else "zig",
             .exclude_extensions = &[_][]const u8{
                 // exclude files from lib/std/compress/testdata
                 ".gz",
@@ -167,6 +195,9 @@ pub fn build(b: *std.Build) !void {
     exe.pie = pie;
     exe.sanitize_thread = sanitize_thread;
     exe.entitlements = entitlements;
+    // TODO -femit-bin/-fno-emit-bin should be inferred by the build system
+    // based on whether or not the exe is run or installed.
+    // https://github.com/ziglang/zig/issues/16351
     if (no_bin) exe.emit_bin = .no_emit;
 
     exe.build_id = b.option(
@@ -175,7 +206,13 @@ pub fn build(b: *std.Build) !void {
         "Request creation of '.note.gnu.build-id' section",
     );
 
-    b.installArtifact(exe);
+    if (!no_bin) {
+        const install_exe = b.addInstallArtifact(exe);
+        if (flat) {
+            install_exe.dest_dir = .prefix;
+        }
+        b.getInstallStep().dependOn(&install_exe.step);
+    }
 
     test_step.dependOn(&exe.step);
 

ci/aarch64-linux-debug.sh

@@ -40,6 +40,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 # Now cmake will use zig as the C/C++ compiler. We reset the environment variables
@@ -49,16 +50,18 @@ unset CXX
 
 ninja install
 
+# TODO: move this to a build.zig step (check-fmt)
 echo "Looking for non-conforming code formatting..."
 stage3-debug/bin/zig fmt --check .. \
   --exclude ../test/cases/ \
   --exclude ../build-debug
 
 # simultaneously test building self-hosted without LLVM and with 32-bit arm
-stage3-debug/bin/zig build -Dtarget=arm-linux-musleabihf
+stage3-debug/bin/zig build \
+  -Dtarget=arm-linux-musleabihf \
+  -Dno-lib
 
 # TODO: add -fqemu back to this line
-
 stage3-debug/bin/zig build test docs \
   --maxrss 24696061952 \
   -fwasmtime \
@@ -68,10 +71,8 @@ stage3-debug/bin/zig build test docs \
   --zig-lib-dir "$(pwd)/../lib"
 
 # Look for HTML errors.
-tidy --drop-empty-elements no -qe "stage3-debug/doc/langref.html"
-
-# Produce the experimental std lib documentation.
-stage3-debug/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
+# TODO: move this to a build.zig flag (-Denable-tidy)
+tidy --drop-empty-elements no -qe "zig-out/doc/langref.html"
 
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-debug/bin/zig build update-zig1
@@ -91,6 +92,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 unset CC
@@ -102,6 +104,7 @@ stage3/bin/zig test ../test/behavior.zig -I../test
 stage3/bin/zig build -p stage4 \
   -Dstatic-llvm \
   -Dtarget=native-native-musl \
+  -Dno-lib \
   --search-prefix "$PREFIX" \
   --zig-lib-dir "$(pwd)/../lib"
 stage4/bin/zig test ../test/behavior.zig -I../test

ci/aarch64-linux-release.sh

@@ -40,6 +40,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 # Now cmake will use zig as the C/C++ compiler. We reset the environment variables
@@ -49,16 +50,18 @@ unset CXX
 
 ninja install
 
+# TODO: move this to a build.zig step (check-fmt)
 echo "Looking for non-conforming code formatting..."
 stage3-release/bin/zig fmt --check .. \
   --exclude ../test/cases/ \
   --exclude ../build-release
 
 # simultaneously test building self-hosted without LLVM and with 32-bit arm
-stage3-release/bin/zig build -Dtarget=arm-linux-musleabihf
+stage3-release/bin/zig build \
+  -Dtarget=arm-linux-musleabihf \
+  -Dno-lib
 
 # TODO: add -fqemu back to this line
-
 stage3-release/bin/zig build test docs \
   --maxrss 24696061952 \
   -fwasmtime \
@@ -68,10 +71,8 @@ stage3-release/bin/zig build test docs \
   --zig-lib-dir "$(pwd)/../lib"
 
 # Look for HTML errors.
-tidy --drop-empty-elements no -qe "stage3-release/doc/langref.html"
-
-# Produce the experimental std lib documentation.
-stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
+# TODO: move this to a build.zig flag (-Denable-tidy)
+tidy --drop-empty-elements no -qe "zig-out/doc/langref.html"
 
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-release/bin/zig build update-zig1
@@ -91,6 +92,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 unset CC
@@ -102,6 +104,7 @@ stage3/bin/zig test ../test/behavior.zig -I../test
 stage3/bin/zig build -p stage4 \
   -Dstatic-llvm \
   -Dtarget=native-native-musl \
+  -Dno-lib \
   --search-prefix "$PREFIX" \
   --zig-lib-dir "$(pwd)/../lib"
 stage4/bin/zig test ../test/behavior.zig -I../test

ci/aarch64-macos-debug.sh

@@ -39,6 +39,7 @@ PATH="$HOME/local/bin:$PATH" cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 $HOME/local/bin/ninja install
@@ -49,6 +50,3 @@ stage3-debug/bin/zig build test docs \
   -Dstatic-llvm \
   -Dskip-non-native \
   --search-prefix "$PREFIX"
-
-# Produce the experimental std lib documentation.
-stage3-debug/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib

ci/aarch64-macos-release.sh

@@ -39,6 +39,7 @@ PATH="$HOME/local/bin:$PATH" cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 $HOME/local/bin/ninja install
@@ -50,9 +51,6 @@ stage3-release/bin/zig build test docs \
   -Dskip-non-native \
   --search-prefix "$PREFIX"
 
-# Produce the experimental std lib documentation.
-stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
-
 # Ensure that stage3 and stage4 are byte-for-byte identical.
 stage3-release/bin/zig build \
   --prefix stage4-release \

ci/aarch64-windows.ps1

@@ -55,7 +55,8 @@ $Env:ZIG_LOCAL_CACHE_DIR="$(Get-Location)\zig-local-cache"
   -DZIG_AR_WORKAROUND=ON `
   -DZIG_TARGET_TRIPLE="$TARGET" `
   -DZIG_TARGET_MCPU="$MCPU" `
-  -DZIG_STATIC=ON
+  -DZIG_STATIC=ON `
+  -DZIG_NO_LIB=ON
 CheckLastExitCode
 
 ninja install
@@ -69,11 +70,3 @@ Write-Output "Main test suite..."
   -Dskip-non-native `
   -Denable-symlinks-windows
 CheckLastExitCode
-
-Write-Output "Testing Autodocs..."
-& "stage3-release\bin\zig.exe" test "..\lib\std\std.zig" `
-  --zig-lib-dir "$ZIG_LIB_DIR" `
-  -femit-docs `
-  -fno-emit-bin
-CheckLastExitCode
-

ci/x86_64-linux-debug.sh

@@ -40,6 +40,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 # Now cmake will use zig as the C/C++ compiler. We reset the environment variables
@@ -49,13 +50,16 @@ unset CXX
 
 ninja install
 
+# TODO: move this to a build.zig step (check-fmt)
 echo "Looking for non-conforming code formatting..."
 stage3-debug/bin/zig fmt --check .. \
   --exclude ../test/cases/ \
   --exclude ../build-debug
 
 # simultaneously test building self-hosted without LLVM and with 32-bit arm
-stage3-debug/bin/zig build -Dtarget=arm-linux-musleabihf
+stage3-debug/bin/zig build \
+  -Dtarget=arm-linux-musleabihf \
+  -Dno-lib
 
 stage3-debug/bin/zig build test docs \
   --maxrss 21000000000 \
@@ -67,10 +71,8 @@ stage3-debug/bin/zig build test docs \
   --zig-lib-dir "$(pwd)/../lib"
 
 # Look for HTML errors.
-tidy --drop-empty-elements no -qe "stage3-debug/doc/langref.html"
-
-# Produce the experimental std lib documentation.
-stage3-debug/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
+# TODO: move this to a build.zig flag (-Denable-tidy)
+tidy --drop-empty-elements no -qe "zig-out/doc/langref.html"
 
 # Ensure that updating the wasm binary from this commit will result in a viable build.
 stage3-debug/bin/zig build update-zig1
@@ -90,6 +92,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 unset CC
@@ -101,6 +104,7 @@ stage3/bin/zig test ../test/behavior.zig -I../test
 stage3/bin/zig build -p stage4 \
   -Dstatic-llvm \
   -Dtarget=native-native-musl \
+  -Dno-lib \
   --search-prefix "$PREFIX" \
   --zig-lib-dir "$(pwd)/../lib"
 stage4/bin/zig test ../test/behavior.zig -I../test

ci/x86_64-linux-release.sh

@@ -40,6 +40,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 # Now cmake will use zig as the C/C++ compiler. We reset the environment variables
@@ -49,14 +50,17 @@ unset CXX
 
 ninja install
 
+# TODO: move this to a build.zig step (check-fmt)
 echo "Looking for non-conforming code formatting..."
 stage3-release/bin/zig fmt --check .. \
   --exclude ../test/cases/ \
   --exclude ../build-debug \
   --exclude ../build-release
 
 # simultaneously test building self-hosted without LLVM and with 32-bit arm
-stage3-release/bin/zig build -Dtarget=arm-linux-musleabihf
+stage3-release/bin/zig build \
+  -Dtarget=arm-linux-musleabihf \
+  -Dno-lib
 
 stage3-release/bin/zig build test docs \
   --maxrss 21000000000 \
@@ -68,10 +72,8 @@ stage3-release/bin/zig build test docs \
   --zig-lib-dir "$(pwd)/../lib"
 
 # Look for HTML errors.
-tidy --drop-empty-elements no -qe "stage3-release/doc/langref.html"
-
-# Produce the experimental std lib documentation.
-stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
+# TODO: move this to a build.zig flag (-Denable-tidy)
+tidy --drop-empty-elements no -qe "zig-out/doc/langref.html"
 
 # Ensure that stage3 and stage4 are byte-for-byte identical.
 stage3-release/bin/zig build \
@@ -107,6 +109,7 @@ cmake .. \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
   -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
   -GNinja
 
 unset CC
@@ -118,6 +121,7 @@ stage3/bin/zig test ../test/behavior.zig -I../test
 stage3/bin/zig build -p stage4 \
   -Dstatic-llvm \
   -Dtarget=native-native-musl \
+  -Dno-lib \
   --search-prefix "$PREFIX" \
   --zig-lib-dir "$(pwd)/../lib"
 stage4/bin/zig test ../test/behavior.zig -I../test

ci/x86_64-macos-release.sh

@@ -43,7 +43,8 @@ cmake .. \
   -DCMAKE_CXX_COMPILER="$ZIG;c++;-target;$TARGET;-mcpu=$MCPU" \
   -DZIG_TARGET_TRIPLE="$TARGET" \
   -DZIG_TARGET_MCPU="$MCPU" \
-  -DZIG_STATIC=ON
+  -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON
 
 make $JOBS install
 
@@ -54,9 +55,6 @@ stage3/bin/zig build test docs \
   -Dskip-non-native \
   --search-prefix "$PREFIX"
 
-# Produce the experimental std lib documentation.
-stage3/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib
-
 # Ensure that stage3 and stage4 are byte-for-byte identical.
 stage3/bin/zig build \
   --prefix stage4 \