diff --git a/nixos/release-small.nix b/nixos/release-small.nix index 7be300bbcf3be..7659d61709ac1 100644 --- a/nixos/release-small.nix +++ b/nixos/release-small.nix @@ -81,7 +81,8 @@ in rec { stdenv subversion tarball - vim; + vim + tests-stdenv-gcc-stageCompare; }; tested = let @@ -131,6 +132,7 @@ in rec { "nixos.tests.proxy" "nixos.tests.simple" "nixpkgs.jdk" + "nixpkgs.tests-stdenv-gcc-stageCompare" ]) ]; }; diff --git a/pkgs/applications/editors/emacs/generic.nix b/pkgs/applications/editors/emacs/generic.nix index 7b71fccd6b8a8..4cad0f12cffef 100644 --- a/pkgs/applications/editors/emacs/generic.nix +++ b/pkgs/applications/editors/emacs/generic.nix @@ -62,9 +62,17 @@ assert withXwidgets -> withGTK3 && webkitgtk != null; assert withTreeSitter -> tree-sitter != null; +let + libGccJitLibraryPaths = [ + "${lib.getLib libgccjit}/lib/gcc" + "${lib.getLib stdenv.cc.libc}/lib" + ] ++ lib.optionals (stdenv.cc?cc.libgcc) [ + "${lib.getLib stdenv.cc.cc.libgcc}/lib" + ]; +in (if withMacport then llvmPackages_6.stdenv else stdenv).mkDerivation (finalAttrs: (lib.optionalAttrs nativeComp { NATIVE_FULL_AOT = "1"; - LIBRARY_PATH = "${lib.getLib stdenv.cc.libc}/lib"; + LIBRARY_PATH = lib.concatStringsSep ":" libGccJitLibraryPaths; } // { pname = pname + lib.optionalString ( !withX && !withNS && !withMacport && !withGTK2 && !withGTK3 ) "-nox"; inherit version; @@ -75,17 +83,15 @@ assert withTreeSitter -> tree-sitter != null; then ./native-comp-driver-options-28.patch else ./native-comp-driver-options.patch; backendPath = (lib.concatStringsSep " " - (builtins.map (x: ''"-B${x}"'') [ + (builtins.map (x: ''"-B${x}"'') ([ # Paths necessary so the JIT compiler finds its libraries: "${lib.getLib libgccjit}/lib" - "${lib.getLib libgccjit}/lib/gcc" - "${lib.getLib stdenv.cc.libc}/lib" - + ] ++ libGccJitLibraryPaths ++ [ # Executable paths necessary for compilation (ld, as): "${lib.getBin stdenv.cc.cc}/bin" "${lib.getBin stdenv.cc.bintools}/bin" "${lib.getBin stdenv.cc.bintools.bintools}/bin" - ])); + ]))); }) ]; diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 96fd8b66bb428..25481fec664da 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -17,9 +17,40 @@ , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null , buildPackages ? {} , libcxx ? null + +# Whether or not to add `-B` and `-L` to `nix-support/cc-{c,ld}flags` +, useCcForLibs ? + + # Always add these flags for Clang, because in order to compile (most + # software) it needs libraries that are shipped and compiled with gcc. + if isClang then true + + # Never add these flags for a build!=host cross-compiler or a host!=target + # ("cross-built-native") compiler; currently nixpkgs has a special build + # path for these (`crossStageStatic`). Hopefully at some point that build + # path will be merged with this one and this conditional will be removed. + else if (with stdenvNoCC; buildPlatform != hostPlatform || hostPlatform != targetPlatform) then false + + # Never add these flags when wrapping the bootstrapFiles' compiler; it has a + # /usr/-like layout with everything smashed into a single outpath, so it has + # no trouble finding its own libraries. + else if (cc.passthru.isFromBootstrapFiles or false) then false + + # Add these flags when wrapping `xgcc` (the first compiler that nixpkgs builds) + else if (cc.passthru.isXgcc or false) then true + + # Add these flags when wrapping `stdenv.cc` + else if (cc.stdenv.cc.cc.passthru.isXgcc or false) then true + + # Do not add these flags in any other situation. This is `false` mainly to + # prevent these flags from being added when wrapping *old* versions of gcc + # (e.g. `gcc6Stdenv`), since they will cause the old gcc to get `-B` and + # `-L` flags pointing at the new gcc's libstdc++ headers. Example failure: + # https://hydra.nixos.org/build/213125495 + else false + +# the derivation at which the `-B` and `-L` flags added by `useCcForLibs` will point , gccForLibs ? if useCcForLibs then cc else null -# same as `gccForLibs`, but generalized beyond clang -, useCcForLibs ? isClang }: with lib; @@ -319,7 +350,7 @@ stdenv.mkDerivation { && targetPlatform.isLinux && !(stdenv.targetPlatform.useAndroidPrebuilt or false) && !(stdenv.targetPlatform.useLLVM or false) - && gccForLibs != null) '' + && gccForLibs != null) ('' echo "--gcc-toolchain=${gccForLibs}" >> $out/nix-support/cc-cflags # Pull in 'cc.out' target to get 'libstdc++fs.a'. It should be in @@ -327,6 +358,11 @@ stdenv.mkDerivation { # TODO(trofi): remove once gcc is fixed to move libraries to .lib output. echo "-L${gccForLibs}/${optionalString (targetPlatform != hostPlatform) "/${targetPlatform.config}"}/lib" >> $out/nix-support/cc-ldflags '' + # this ensures that when clang passes -lgcc_s to lld (as it does + # when building e.g. firefox), lld is able to find libgcc_s.so + + lib.optionalString (gccForLibs?libgcc) '' + echo "-L${gccForLibs.libgcc}/lib" >> $out/nix-support/cc-ldflags + '') ## ## General libc support diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix index 3f6a224fa6cd6..bd7702ebb9162 100644 --- a/pkgs/build-support/emacs/wrapper.nix +++ b/pkgs/build-support/emacs/wrapper.nix @@ -67,10 +67,12 @@ runCommand # Store all paths we want to add to emacs here, so that we only need to add # one path to the load lists deps = runCommand "emacs-packages-deps" - { + ({ inherit explicitRequires lndir emacs; nativeBuildInputs = lib.optional nativeComp gcc; - } + } // lib.optionalAttrs nativeComp { + inherit (emacs) LIBRARY_PATH; + }) '' findInputsOld() { local pkg="$1"; shift diff --git a/pkgs/build-support/trivial-builders/test/references.nix b/pkgs/build-support/trivial-builders/test/references.nix index 3e1eb16eecd0b..7c8ea83f3c8bf 100644 --- a/pkgs/build-support/trivial-builders/test/references.nix +++ b/pkgs/build-support/trivial-builders/test/references.nix @@ -45,10 +45,8 @@ testers.nixosTest { """) ''; meta = { - license = lib.licenses.mit; # nixpkgs license maintainers = with lib.maintainers; [ roberth ]; - description = "Run the Nixpkgs trivial builders tests"; }; } diff --git a/pkgs/development/compilers/gcc/11/default.nix b/pkgs/development/compilers/gcc/11/default.nix index b2330abd4b75b..323e68e8691ad 100644 --- a/pkgs/development/compilers/gcc/11/default.nix +++ b/pkgs/development/compilers/gcc/11/default.nix @@ -28,6 +28,8 @@ , buildPackages , libxcrypt , disableGdbPlugin ? !enablePlugin +, nukeReferences +, callPackage }: # Make sure we get GNU sed. @@ -49,7 +51,7 @@ with builtins; let majorVersion = "11"; version = "${majorVersion}.3.0"; - disableBootstrap = !(with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform); + disableBootstrap = true; inherit (stdenv) buildPlatform hostPlatform targetPlatform; @@ -159,7 +161,7 @@ let majorVersion = "11"; in -stdenv.mkDerivation ({ +lib.pipe (stdenv.mkDerivation ({ pname = "${crossNameAddon}${name}"; inherit version; @@ -250,9 +252,8 @@ stdenv.mkDerivation ({ targetConfig = if targetPlatform != hostPlatform then targetPlatform.config else null; buildFlags = - let target = - lib.optionalString (profiledCompiler) "profiled" + - lib.optionalString (targetPlatform == hostPlatform && hostPlatform == buildPlatform && !disableBootstrap) "bootstrap"; + let target = lib.optionalString (profiledCompiler) "profiled" + + lib.optionalString (targetPlatform == hostPlatform && hostPlatform == buildPlatform && !disableBootstrap) "bootstrap"; in lib.optional (target != "") target; inherit (callFile ../common/strip-attributes.nix { }) @@ -310,4 +311,8 @@ stdenv.mkDerivation ({ } // optionalAttrs (enableMultilib) { dontMoveLib64 = true; } -) +)) +[ + (callPackage ../common/libgcc.nix { inherit langC langCC langJit; }) + (callPackage ../common/checksum.nix { inherit langC langCC; }) +] diff --git a/pkgs/development/compilers/gcc/12/default.nix b/pkgs/development/compilers/gcc/12/default.nix index 78dc30a34463e..d782eedcbea85 100644 --- a/pkgs/development/compilers/gcc/12/default.nix +++ b/pkgs/development/compilers/gcc/12/default.nix @@ -29,6 +29,8 @@ , buildPackages , libxcrypt , disableGdbPlugin ? !enablePlugin +, nukeReferences +, callPackage }: # Make sure we get GNU sed. @@ -54,7 +56,7 @@ with builtins; let majorVersion = "12"; version = "${majorVersion}.2.0"; - disableBootstrap = !(with stdenv; targetPlatform == hostPlatform && hostPlatform == buildPlatform); + disableBootstrap = true; inherit (stdenv) buildPlatform hostPlatform targetPlatform; @@ -177,6 +179,7 @@ let majorVersion = "12"; mpfr name noSysDirs + nukeReferences patchelf perl profiledCompiler @@ -194,7 +197,7 @@ let majorVersion = "12"; in -stdenv.mkDerivation ({ +lib.pipe (stdenv.mkDerivation ({ pname = "${crossNameAddon}${name}"; inherit version; @@ -344,4 +347,9 @@ stdenv.mkDerivation ({ } // optionalAttrs (enableMultilib) { dontMoveLib64 = true; } -) +)) +[ + (callPackage ../common/libgcc.nix { inherit langC langCC langJit; }) + (callPackage ../common/checksum.nix { inherit langC langCC; }) +] + diff --git a/pkgs/development/compilers/gcc/common/checksum.nix b/pkgs/development/compilers/gcc/common/checksum.nix new file mode 100644 index 0000000000000..b3e387b107baf --- /dev/null +++ b/pkgs/development/compilers/gcc/common/checksum.nix @@ -0,0 +1,40 @@ +{ lib +, stdenv +, nukeReferences +, langC +, langCC +, runtimeShell +}: + +let + enableChecksum = (with stdenv; buildPlatform == hostPlatform && hostPlatform == targetPlatform) && langC && langCC; +in +(pkg: pkg.overrideAttrs (previousAttrs: lib.optionalAttrs enableChecksum { + outputs = previousAttrs.outputs ++ lib.optionals enableChecksum [ "checksum" ]; + # This is a separate phase because gcc assembles its phase scripts + # in bash instead of nix (we should fix that). + preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "postInstallSaveChecksumPhase" ]; + # + # gcc uses an auxiliary utility `genchecksum` to md5-hash (most of) its + # `.o` and `.a` files prior to linking (in case the linker is + # nondeterministic). Since we want to compare across gccs built from two + # separate derivations, we wrap `genchecksum` with a `nuke-references` + # call. We also stash copies of the inputs to `genchecksum` in + # `$checksum/inputs/` -- this is extremely helpful for debugging since + # it's hard to get Nix to not delete the $NIX_BUILD_TOP of a successful + # build. + # + postInstallSaveChecksumPhase = '' + mv gcc/build/genchecksum gcc/build/.genchecksum-wrapped + cat > gcc/build/genchecksum <<\EOF + #!${runtimeShell} + ${nukeReferences}/bin/nuke-refs $@ + for INPUT in "$@"; do install -Dt $INPUT $checksum/inputs/; done + exec build/.genchecksum-wrapped $@ + EOF + chmod +x gcc/build/genchecksum + rm gcc/*-checksum.* + make -C gcc cc1-checksum.o cc1plus-checksum.o + install -Dt $checksum/checksums/ gcc/cc*-checksum.o + ''; +})) diff --git a/pkgs/development/compilers/gcc/common/libgcc.nix b/pkgs/development/compilers/gcc/common/libgcc.nix new file mode 100644 index 0000000000000..e352393601962 --- /dev/null +++ b/pkgs/development/compilers/gcc/common/libgcc.nix @@ -0,0 +1,96 @@ +{ lib +, stdenv +, langC +, langCC +, langJit +}: + +let + enableLibGccOutput = (with stdenv; targetPlatform == hostPlatform) && !langJit; +in +(pkg: pkg.overrideAttrs (previousAttrs: lib.optionalAttrs ((!langC) || langJit || enableLibGccOutput) { + outputs = previousAttrs.outputs ++ lib.optionals enableLibGccOutput [ "libgcc" ]; + # This is a separate phase because gcc assembles its phase scripts + # in bash instead of nix (we should fix that). + preFixupPhases = (previousAttrs.preFixupPhases or []) ++ [ "preFixupLibGccPhase" ]; + preFixupLibGccPhase = + # delete extra/unused builds of libgcc_s in non-langC builds + # (i.e. libgccjit, gnat, etc) to avoid potential confusion + lib.optionalString (!langC) '' + rm -f $out/lib/libgcc_s.so* + '' + + # TODO(amjoseph): remove the `libgcc_s.so` symlinks below and replace them + # with a `-L${gccForLibs.libgcc}/lib` in cc-wrapper's + # `$out/nix-support/cc-flags`. See also: + # - https://github.com/NixOS/nixpkgs/pull/209870#discussion_r1130614895 + # - https://github.com/NixOS/nixpkgs/pull/209870#discussion_r1130635982 + # - https://github.com/NixOS/nixpkgs/commit/404155c6acfa59456aebe6156b22fe385e7dec6f + # + # move `libgcc_s.so` into its own output, `$libgcc` + + lib.optionalString enableLibGccOutput ('' + # move libgcc from lib to its own output (libgcc) + mkdir -p $libgcc/lib + mv $lib/lib/libgcc_s.so $libgcc/lib/ + mv $lib/lib/libgcc_s.so.1 $libgcc/lib/ + ln -s $libgcc/lib/libgcc_s.so $lib/lib/ + ln -s $libgcc/lib/libgcc_s.so.1 $lib/lib/ + '' + # + # Nixpkgs ordinarily turns dynamic linking into pseudo-static linking: + # libraries are still loaded dynamically, exactly which copy of each + # library is loaded is permanently fixed at compile time (via RUNPATH). + # For libgcc_s we must revert to the "impure dynamic linking" style found + # in imperative software distributions. We must do this because + # `libgcc_s` calls `malloc()` and therefore has a `DT_NEEDED` for `libc`, + # which creates two problems: + # + # 1. A circular package dependency `glibc`<-`libgcc`<-`glibc` + # + # 2. According to the `-Wl,-rpath` flags added by Nixpkgs' `ld-wrapper`, + # the two versions of `glibc` in the cycle above are actually + # different packages. The later one is compiled by this `gcc`, but + # the earlier one was compiled by the compiler *that compiled* this + # `gcc` (usually the bootstrapFiles). In any event, the `glibc` + # dynamic loader won't honor that specificity without namespaced + # manual loads (`dlmopen()`). Once a `libc` is present in the address + # space of a process, that `libc` will be used to satisfy all + # `DT_NEEDED`s for `libc`, regardless of `RUNPATH`s. + # + # So we wipe the RUNPATH using `patchelf --set-rpath ""`. We can't use + # `patchelf --remove-rpath`, because at least as of patchelf 0.15.0 it + # will leave the old RUNPATH string in the file where the reference + # scanner can still find it: + # + # https://github.com/NixOS/patchelf/issues/453 + # + # Note: we might be using the bootstrapFiles' copy of patchelf, so we have + # to keep doing it this way until both the issue is fixed *and* all the + # bootstrapFiles are regenerated, on every platform. + # + # This patchelfing is *not* effectively equivalent to copying + # `libgcc_s` into `glibc`'s outpath. There is one minor and one + # major difference: + # + # 1. (Minor): multiple builds of `glibc` (say, with different + # overrides or parameters) will all reference a single store + # path: + # + # /nix/store/xxx...xxx-gcc-libgcc/lib/libgcc_s.so.1 + # + # This many-to-one referrer relationship will be visible in the store's + # dependency graph, and will be available to `nix-store -q` queries. + # Copying `libgcc_s` into each of its referrers would lose that + # information. + # + # 2. (Major): by referencing `libgcc_s.so.1`, rather than copying it, we + # are still able to run `nix-store -qd` on it to find out how it got + # built! Most importantly, we can see from that deriver which compiler + # was used to build it (or if it is part of the unpacked + # bootstrap-files). Copying `libgcc_s.so.1` from one outpath to + # another eliminates the ability to make these queries. + # + + '' + patchelf --set-rpath "" $libgcc/lib/libgcc_s.so.1 + ''); +})) diff --git a/pkgs/development/libraries/glibc/default.nix b/pkgs/development/libraries/glibc/default.nix index e1a427e35b658..8ad9c90ff7ac7 100644 --- a/pkgs/development/libraries/glibc/default.nix +++ b/pkgs/development/libraries/glibc/default.nix @@ -66,33 +66,26 @@ in ]); }; - # When building glibc from bootstrap-tools, we need libgcc_s at RPATH for - # any program we run, because the gcc will have been placed at a new - # store path than that determined when built (as a source for the - # bootstrap-tools tarball) - # Building from a proper gcc staying in the path where it was installed, - # libgcc_s will now be at {gcc}/lib, and gcc's libgcc will be found without - # any special hack. - # TODO: remove this hack. Things that rely on this hack today: - # - dejagnu: during linux bootstrap tcl SIGSEGVs - # - clang-wrapper in cross-compilation - # Last attempt: https://github.com/NixOS/nixpkgs/pull/36948 - preInstall = lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' - if [ -f ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so.1 ]; then - mkdir -p $out/lib - cp ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so.1 $out/lib/libgcc_s.so.1 - # the .so It used to be a symlink, but now it is a script - cp -a ${lib.getLib stdenv.cc.cc}/lib/libgcc_s.so $out/lib/libgcc_s.so - # wipe out reference to previous libc it was built against - chmod +w $out/lib/libgcc_s.so.1 - # rely on default RUNPATHs of the binary and other libraries - # Do no force-pull wrong glibc. - patchelf --remove-rpath $out/lib/libgcc_s.so.1 - # 'patchelf' does not remove the string itself. Wipe out - # string reference to avoid possible link to bootstrapTools - ${buildPackages.nukeReferences}/bin/nuke-refs $out/lib/libgcc_s.so.1 - fi - ''; + # glibc needs to `dlopen()` `libgcc_s.so` but does not link + # against it. Furthermore, glibc doesn't use the ordinary + # `dlopen()` call to do this; instead it uses one which ignores + # most paths: + # + # https://sourceware.org/legacy-ml/libc-help/2013-11/msg00026.html + # + # In order to get it to not ignore `libgcc_s.so`, we have to add its path to + # `user-defined-trusted-dirs`: + # + # https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/Makefile;h=b509b3eada1fb77bf81e2a0ca5740b94ad185764#l1355 + # + # Conveniently, this will also inform Nix of the fact that glibc depends on + # gcc.libgcc, since the path will be embedded in the resulting binary. + # + makeFlags = + (previousAttrs.makeFlags or []) + ++ lib.optionals (stdenv.cc.cc?libgcc) [ + "user-defined-trusted-dirs=${stdenv.cc.cc.libgcc}/lib" + ]; postInstall = (if stdenv.hostPlatform == stdenv.buildPlatform then '' echo SUPPORTED-LOCALES=C.UTF-8/UTF-8 > ../glibc-2*/localedata/SUPPORTED @@ -164,6 +157,12 @@ in separateDebugInfo = true; + passthru = + (previousAttrs.passthru or {}) + // lib.optionalAttrs (stdenv.cc.cc?libgcc) { + inherit (stdenv.cc.cc) libgcc; + }; + meta = (previousAttrs.meta or {}) // { description = "The GNU C Library"; }; }) diff --git a/pkgs/development/tools/misc/ccache/default.nix b/pkgs/development/tools/misc/ccache/default.nix index df12d6c2c72cd..ef498f6f6e08b 100644 --- a/pkgs/development/tools/misc/ccache/default.nix +++ b/pkgs/development/tools/misc/ccache/default.nix @@ -74,7 +74,8 @@ stdenv.mkDerivation (finalAttrs: { # A derivation that provides gcc and g++ commands, but that # will end up calling ccache for the given cacheDir links = { unwrappedCC, extraConfig }: stdenv.mkDerivation { - name = "ccache-links"; + pname = "ccache-links"; + inherit (finalAttrs) version; passthru = { isClang = unwrappedCC.isClang or false; isGNU = unwrappedCC.isGNU or false; diff --git a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh index 5b5677eef1366..09bf25f52153f 100644 --- a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh +++ b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh @@ -30,6 +30,13 @@ LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/mv $out/lib/libstdc++.* $LIBSTDCXX_ # use a copy of patchelf. LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/bin/patchelf . +# Older versions of the bootstrap-files did not compile their +# patchelf with -static-libgcc, so we have to be very careful not to +# run patchelf on the same copy of libgcc_s that it links against. +LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/lib/libgcc_s.so.1 . +LD_LIBRARY_PATH=.:$out/lib:$LIBSTDCXX_SO_DIR $LD_BINARY \ + ./patchelf --set-rpath $out/lib --force-rpath $out/lib/libgcc_s.so.1 + for i in $out/bin/* $out/libexec/gcc/*/*/*; do if [ -L "$i" ]; then continue; fi if [ -z "${i##*/liblto*}" ]; then continue; fi diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 51f217f03b276..181936de5ded1 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -10,13 +10,10 @@ # # Goals of the bootstrap process: # 1. final stdenv must not reference any of the bootstrap files. -# 2. final stdenv must not contain any of the bootstrap files -# (the only current violation is libgcc_s.so in glibc). +# 2. final stdenv must not contain any of the bootstrap files. # 3. final stdenv must not contain any of the files directly # generated by the bootstrap code generators (assembler, linker, -# compiler). The only current violations are: libgcc_s.so in glibc, -# the lib{mpfr,mpc,gmp,isl} which are statically linked -# into the final gcc). +# compiler). # # These goals ensure that final packages and final stdenv are built # exclusively using nixpkgs package definitions and don't depend @@ -111,6 +108,21 @@ let isBuiltByBootstrapFilesCompiler = pkg: isFromNixpkgs pkg && isFromBootstrapFiles pkg.stdenv.cc.cc; + commonGccOverrides = { + # Use a deterministically built compiler + # see https://github.com/NixOS/nixpkgs/issues/108475 for context + reproducibleBuild = true; + profiledCompiler = false; + + # It appears that libcc1 (which is not a g++ plugin; it is a gdb plugin) gets linked against + # the libstdc++ from the compiler that *built* g++, not the libstdc++ which was just built. + # This causes a reference chain from stdenv to the bootstrapFiles: + # + # stdenv -> gcc-lib -> xgcc-lib -> bootstrapFiles + # + disableGdbPlugin = true; + }; + commonPreHook = '' export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" @@ -170,7 +182,7 @@ let cc = if prevStage.gcc-unwrapped == null then null - else lib.makeOverridable (import ../../build-support/cc-wrapper) { + else (lib.makeOverridable (import ../../build-support/cc-wrapper) { name = "${name}-gcc-wrapper"; nativeTools = false; nativeLibc = false; @@ -184,7 +196,12 @@ let inherit lib; inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; - }; + }).overrideAttrs(a: lib.optionalAttrs (prevStage.gcc-unwrapped.passthru.isXgcc or false) { + # This affects only `xgcc` (the compiler which compiles the final compiler). + postFixup = (a.postFixup or "") + '' + echo "--sysroot=${lib.getDev (getLibc prevStage)}" >> $out/nix-support/cc-cflags + ''; + }); overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; @@ -226,7 +243,7 @@ in ${localSystem.libc} = self.stdenv.mkDerivation { pname = "bootstrap-stage0-${localSystem.libc}"; strictDeps = true; - version = "bootstrap"; + version = "bootstrapFiles"; enableParallelBuilding = true; buildCommand = '' mkdir -p $out @@ -282,7 +299,7 @@ in }; inherit (prevStage) ccWrapperStdenv - gcc-unwrapped coreutils gnugrep; + gcc-unwrapped coreutils gnugrep binutils; ${localSystem.libc} = getLibc prevStage; @@ -295,6 +312,75 @@ in }; }) + # First rebuild of gcc; this is linked against all sorts of junk + # from the bootstrap-files, but we only care about the code that + # this compiler *emits*. The `gcc` binary produced in this stage + # is not part of the final stdenv. + (prevStage: + assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; + assert isFromBootstrapFiles prevStage."${localSystem.libc}"; + assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { + name = "bootstrap-stage-xgcc"; + overrides = final: prev: { + inherit (prevStage) ccWrapperStdenv coreutils gnugrep gettext bison texinfo zlib gnum4 perl; + patchelf = bootstrapTools; + ${localSystem.libc} = getLibc prevStage; + gmp = prev.gmp.override { cxx = false; }; + gcc-unwrapped = + (prev.gcc-unwrapped.override (commonGccOverrides // { + # The most logical name for this package would be something like + # "gcc-stage1". Unfortunately "stage" is already reserved for the + # layers of stdenv, so using "stage" in the name of this package + # would cause massive confusion. + # + # Gcc calls its "stage1" compiler `xgcc` (--disable-bootstrap results + # in `xgcc` being copied to $prefix/bin/gcc). So we imitate that. + # + name = "xgcc"; + + })).overrideAttrs (a: { + + # This signals to cc-wrapper (as overridden above in this file) to add `--sysroot` + # to `$out/nix-support/cc-cflags`. + passthru = a.passthru // { isXgcc = true; }; + + # Gcc will look for the C library headers in + # + # ${with_build_sysroot}${native_system_header_dir} + # + # The ordinary gcc expression sets `--with-build-sysroot=/` and sets + # `native-system-header-dir` to `"${lib.getDev stdenv.cc.libc}/include`. + # + # Unfortunately the value of "--with-native-system-header-dir=" gets "burned in" to the + # compiler, and it is quite difficult to get the compiler to change or ignore it + # afterwards. On the other hand, the `sysroot` is very easy to change; you can just pass + # a `--sysroot` flag to `gcc`. + # + # So we override the expression to remove the default settings for these flags, and + # replace them such that the concatenated value will be the same as before, but we split + # the value between the two variables differently: `--native-system-header-dir=/include`, + # and `--with-build-sysroot=${lib.getDev stdenv.cc.libc}`. + # + configureFlags = (a.configureFlags or []) ++ [ + "--with-native-system-header-dir=/include" + "--with-build-sysroot=${lib.getDev final.stdenv.cc.libc}" + ]; + + # This is a separate phase because gcc assembles its phase scripts + # in bash instead of nix (we should fix that). + preFixupPhases = (a.preFixupPhases or []) ++ [ "preFixupXgccPhase" ]; + + # This is needed to prevent "error: cycle detected in build of '...-xgcc-....drv' + # in the references of output 'lib' from output 'out'" + preFixupXgccPhase = '' + find $lib/lib/ -name \*.so\* -exec patchelf --shrink-rpath {} \; || true + ''; + }); + }; + }) # 2nd stdenv that contains our own rebuilt binutils and is used for # compiling our own Glibc. @@ -303,7 +389,7 @@ in # previous stage1 stdenv: assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; assert isFromBootstrapFiles prevStage."${localSystem.libc}"; - assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; assert isFromBootstrapFiles prevStage.coreutils; assert isFromBootstrapFiles prevStage.gnugrep; stageFun prevStage { @@ -313,7 +399,7 @@ in inherit (prevStage) ccWrapperStdenv gettext gcc-unwrapped coreutils gnugrep - perl gnum4 bison; + perl gnum4 bison texinfo which; dejagnu = super.dejagnu.overrideAttrs (a: { doCheck = false; } ); # We need libidn2 and its dependency libunistring as glibc dependency. @@ -378,11 +464,12 @@ in # binutils and rest of the bootstrap tools, including GCC. (prevStage: # previous stage2 stdenv: - assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; assert isFromBootstrapFiles prevStage.coreutils; assert isFromBootstrapFiles prevStage.gnugrep; + assert lib.all isBuiltByNixpkgsCompiler (with prevStage; [ gmp isl_0_20 libmpc mpfr ]); stageFun prevStage { name = "bootstrap-stage3"; @@ -390,25 +477,20 @@ in inherit (prevStage) ccWrapperStdenv binutils coreutils gnugrep gettext - perl patchelf linuxHeaders gnum4 bison libidn2 libunistring; + perl patchelf linuxHeaders gnum4 bison libidn2 libunistring libxcrypt; + # We build a special copy of libgmp which doesn't use libstdc++, because + # xgcc++'s libstdc++ references the bootstrap-files (which is what + # compiles xgcc++). + gmp = super.gmp.override { cxx = false; }; + } // { ${localSystem.libc} = getLibc prevStage; - gcc-unwrapped = - let makeStaticLibrariesAndMark = pkg: - lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; }) - .overrideAttrs (a: { pname = "${a.pname}-stage3"; }); - in super.gcc-unwrapped.override { - # Link GCC statically against GMP etc. This makes sense because - # these builds of the libraries are only used by GCC, so it - # reduces the size of the stdenv closure. - gmp = makeStaticLibrariesAndMark super.gmp; - mpfr = makeStaticLibrariesAndMark super.mpfr; - libmpc = makeStaticLibrariesAndMark super.libmpc; - isl = makeStaticLibrariesAndMark super.isl_0_20; - # Use a deterministically built compiler - # see https://github.com/NixOS/nixpkgs/issues/108475 for context - reproducibleBuild = true; - profiledCompiler = false; - }; + gcc-unwrapped = (super.gcc-unwrapped.override (commonGccOverrides // { + inherit (prevStage) which; + } + )).overrideAttrs (a: { + # so we can add them to allowedRequisites below + passthru = a.passthru // { inherit (self) gmp mpfr libmpc isl; }; + }); }; extraNativeBuildInputs = [ prevStage.patchelf ] ++ # Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64. @@ -422,18 +504,11 @@ in # (prevStage: # previous stage3 stdenv: - assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isFromBootstrapFiles prevStage.coreutils; - assert isFromBootstrapFiles prevStage.gnugrep; - # Can assume prevStage.gcc-unwrapped has almost no code from - # bootstrapTools as gcc bootstraps internally. The only - # exceptions are crt files from glibc built bybootstrapTools - # used to link executables and libraries, and the - # bootstrapTools-built, statically-linked - # lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc - # (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d). + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; stageFun prevStage { name = "bootstrap-stage4"; @@ -453,11 +528,6 @@ in }; }; - # force gmp to rebuild so we have the option of dynamically linking - # libgmp without creating a reference path from: - # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap - gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; }); - # To allow users' overrides inhibit dependencies too heavy for # bootstrap, like guile: https://github.com/NixOS/nixpkgs/issues/181188 gnumake = super.gnumake.override { inBootstrap = true; }; @@ -494,11 +564,11 @@ in (prevStage: # previous stage4 stdenv; see stage3 comment regarding gcc, # which applies here as well. - assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isBuiltByNixpkgsCompiler prevStage.coreutils; - assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; { inherit config overlays; stdenv = import ../generic rec { @@ -546,11 +616,15 @@ in ) # More complicated cases ++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] ) - ++ [ /*propagated from .dev*/ linuxHeaders - binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params + ++ [ linuxHeaders # propagated from .dev + binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params gcc.cc.libgcc glibc.passthru.libgcc ] - ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") - [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ]; + ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") + [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ] + ++ (with gcc-unwrapped.passthru; [ + gmp libmpc mpfr isl + ]) + ; overrides = self: super: { inherit (prevStage) @@ -579,10 +653,10 @@ in (prevStage: # previous stage5 stdenv; see stage3 comment regarding gcc, # which applies here as well. - assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isBuiltByNixpkgsCompiler prevStage.coreutils; - assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; { inherit (prevStage) config overlays stdenv; }) ] diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index 3aa7f6a3df537..091130ebf93a8 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -2,6 +2,10 @@ let libc = pkgs.stdenv.cc.libc; + patchelf = pkgs.patchelf.overrideAttrs(previousAttrs: { + NIX_CFLAGS_COMPILE = (previousAttrs.NIX_CFLAGS_COMPILE or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + NIX_CFLAGS_LINK = (previousAttrs.NIX_CFLAGS_LINK or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + }); in with pkgs; rec { @@ -127,7 +131,7 @@ in with pkgs; rec { cp -d ${bootGCC.out}/bin/gcc $out/bin cp -d ${bootGCC.out}/bin/cpp $out/bin cp -d ${bootGCC.out}/bin/g++ $out/bin - cp -d ${bootGCC.lib}/lib/libgcc_s.so* $out/lib + cp ${bootGCC.lib}/lib/libgcc_s.so* $out/lib cp -d ${bootGCC.lib}/lib/libstdc++.so* $out/lib cp -d ${bootGCC.out}/lib/libssp.a* $out/lib cp -d ${bootGCC.out}/lib/libssp_nonshared.a $out/lib @@ -149,6 +153,7 @@ in with pkgs; rec { rm -rf $out/include/c++/*/ext/parallel cp -d ${gmpxx.out}/lib/libgmp*.so* $out/lib + cp -d ${isl.out}/lib/libisl*.so* $out/lib cp -d ${mpfr.out}/lib/libmpfr*.so* $out/lib cp -d ${libmpc.out}/lib/libmpc*.so* $out/lib cp -d ${zlib.out}/lib/libz.so* $out/lib diff --git a/pkgs/test/stdenv/gcc-stageCompare.nix b/pkgs/test/stdenv/gcc-stageCompare.nix new file mode 100644 index 0000000000000..e5c2ed5921b30 --- /dev/null +++ b/pkgs/test/stdenv/gcc-stageCompare.nix @@ -0,0 +1,32 @@ +# This test *must* be run prior to releasing any build of either stdenv or the +# gcc that it exports! This check should also be part of CI for any PR that +# causes a rebuild of `stdenv.cc`. +# +# When we used gcc's internal bootstrap it did this check as part of (and +# serially with) the gcc derivation. Now that we bootstrap externally this +# check can be done in parallel with any/all of stdenv's referrers. But we +# must remember to do the check. +# + +{ stdenv +, pkgs +, lib +}: + +assert stdenv.cc.isGNU; +with pkgs; +# rebuild gcc using the "final" stdenv +let gcc-stageCompare = (gcc-unwrapped.override { + reproducibleBuild = true; + profiledCompiler = false; + stdenv = overrideCC stdenv (wrapCCWith { + cc = stdenv.cc; + }); + }).overrideAttrs(_: { + NIX_OUTPATH_USED_AS_RANDOM_SEED = stdenv.cc.cc.out; + }); +in (runCommand "gcc-stageCompare" {} '' + diff -sr ${pkgs.gcc-unwrapped.checksum}/checksums ${gcc-stageCompare.checksum}/checksums && touch $out +'').overrideAttrs (a: { + meta = (a.meta or { }) // { platforms = lib.platforms.linux; }; +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 43e63c611916a..c5749c18f55c3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -33898,6 +33898,8 @@ with pkgs; testssl = callPackage ../applications/networking/testssl { }; + tests-stdenv-gcc-stageCompare = callPackage ../test/stdenv/gcc-stageCompare.nix { }; + lavalauncher = callPackage ../applications/misc/lavalauncher { }; t-rec = callPackage ../misc/t-rec { diff --git a/pkgs/top-level/pkg-config/test-defaultPkgConfigPackages.nix b/pkgs/top-level/pkg-config/test-defaultPkgConfigPackages.nix index 37687117987d0..108a2b67504ff 100644 --- a/pkgs/top-level/pkg-config/test-defaultPkgConfigPackages.nix +++ b/pkgs/top-level/pkg-config/test-defaultPkgConfigPackages.nix @@ -4,7 +4,8 @@ let inherit (lib.strings) escapeNixIdentifier; - allTests = lib.mapAttrs (k: v: if v == null then null else makePkgConfigTestMaybe k v) defaultPkgConfigPackages; + allTests = lib.mapAttrs (k: v: if v == null then null else makePkgConfigTestMaybe k v) + (builtins.removeAttrs defaultPkgConfigPackages ["recurseForDerivations"]); # nix-build rejects attribute names with periods # This will build those regardless. diff --git a/pkgs/top-level/release-small.nix b/pkgs/top-level/release-small.nix index 7bf4a234bb657..333e285cef0a0 100644 --- a/pkgs/top-level/release-small.nix +++ b/pkgs/top-level/release-small.nix @@ -150,5 +150,5 @@ with import ./release-lib.nix { inherit supportedSystems nixpkgsArgs; }; xfsprogs = linux; xkeyboard_config = linux; zip = all; - + tests-stdenv-gcc-stageCompare = all; } ))