diff options
author | Bernardo Meurer <bernardo@meurer.org> | 2023-04-03 08:19:03 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-04-03 08:19:03 -0700 |
commit | f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc (patch) | |
tree | 791eb576b729778367f1611ee7071419ea6aeb02 /pkgs/stdenv/linux | |
parent | 89ab0e405c1ca4bcc480f8ab1f0d740376e3ee09 (diff) | |
parent | 6c209e862e18d6a9d103a80d5fa2443f4e47163e (diff) | |
download | nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar.gz nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar.bz2 nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar.lz nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar.xz nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.tar.zst nixpkgs-f1f6ca8bcd8d85a22a6f3e3d02c4e0bf9cb2c9cc.zip |
Merge pull request #209870 from amjoseph-nixpkgs/pr/stdenv/external-gcc-bootstrap
Diffstat (limited to 'pkgs/stdenv/linux')
-rw-r--r-- | pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh | 7 | ||||
-rw-r--r-- | pkgs/stdenv/linux/default.nix | 200 | ||||
-rw-r--r-- | pkgs/stdenv/linux/make-bootstrap-tools.nix | 7 |
3 files changed, 150 insertions, 64 deletions
diff --git a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh index 5b5677eef13..09bf25f5215 100644 --- a/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh +++ b/pkgs/stdenv/linux/bootstrap-tools/scripts/unpack-bootstrap-tools.sh @@ -30,6 +30,13 @@ LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/mv $out/lib/libstdc++.* $LIBSTDCXX_ # use a copy of patchelf. LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/bin/patchelf . +# Older versions of the bootstrap-files did not compile their +# patchelf with -static-libgcc, so we have to be very careful not to +# run patchelf on the same copy of libgcc_s that it links against. +LD_LIBRARY_PATH=$out/lib $LD_BINARY $out/bin/cp $out/lib/libgcc_s.so.1 . +LD_LIBRARY_PATH=.:$out/lib:$LIBSTDCXX_SO_DIR $LD_BINARY \ + ./patchelf --set-rpath $out/lib --force-rpath $out/lib/libgcc_s.so.1 + for i in $out/bin/* $out/libexec/gcc/*/*/*; do if [ -L "$i" ]; then continue; fi if [ -z "${i##*/liblto*}" ]; then continue; fi diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 3f2d77729ab..ca6d720fe77 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -10,13 +10,10 @@ # # Goals of the bootstrap process: # 1. final stdenv must not reference any of the bootstrap files. -# 2. final stdenv must not contain any of the bootstrap files -# (the only current violation is libgcc_s.so in glibc). +# 2. final stdenv must not contain any of the bootstrap files. # 3. final stdenv must not contain any of the files directly # generated by the bootstrap code generators (assembler, linker, -# compiler). The only current violations are: libgcc_s.so in glibc, -# the lib{mpfr,mpc,gmp,isl} which are statically linked -# into the final gcc). +# compiler). # # These goals ensure that final packages and final stdenv are built # exclusively using nixpkgs package definitions and don't depend @@ -111,6 +108,21 @@ let isBuiltByBootstrapFilesCompiler = pkg: isFromNixpkgs pkg && isFromBootstrapFiles pkg.stdenv.cc.cc; + commonGccOverrides = { + # Use a deterministically built compiler + # see https://github.com/NixOS/nixpkgs/issues/108475 for context + reproducibleBuild = true; + profiledCompiler = false; + + # It appears that libcc1 (which is not a g++ plugin; it is a gdb plugin) gets linked against + # the libstdc++ from the compiler that *built* g++, not the libstdc++ which was just built. + # This causes a reference chain from stdenv to the bootstrapFiles: + # + # stdenv -> gcc-lib -> xgcc-lib -> bootstrapFiles + # + disableGdbPlugin = true; + }; + commonPreHook = '' export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}" @@ -170,7 +182,7 @@ let cc = if prevStage.gcc-unwrapped == null then null - else lib.makeOverridable (import ../../build-support/cc-wrapper) { + else (lib.makeOverridable (import ../../build-support/cc-wrapper) { name = "${name}-gcc-wrapper"; nativeTools = false; nativeLibc = false; @@ -184,7 +196,12 @@ let inherit lib; inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; - }; + }).overrideAttrs(a: lib.optionalAttrs (prevStage.gcc-unwrapped.passthru.isXgcc or false) { + # This affects only `xgcc` (the compiler which compiles the final compiler). + postFixup = (a.postFixup or "") + '' + echo "--sysroot=${lib.getDev (getLibc prevStage)}" >> $out/nix-support/cc-cflags + ''; + }); overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; @@ -226,7 +243,7 @@ in ${localSystem.libc} = self.stdenv.mkDerivation { pname = "bootstrap-stage0-${localSystem.libc}"; strictDeps = true; - version = "bootstrap"; + version = "bootstrapFiles"; enableParallelBuilding = true; buildCommand = '' mkdir -p $out @@ -282,7 +299,7 @@ in }; inherit (prevStage) ccWrapperStdenv - gcc-unwrapped coreutils gnugrep; + gcc-unwrapped coreutils gnugrep binutils; ${localSystem.libc} = getLibc prevStage; @@ -295,6 +312,75 @@ in }; }) + # First rebuild of gcc; this is linked against all sorts of junk + # from the bootstrap-files, but we only care about the code that + # this compiler *emits*. The `gcc` binary produced in this stage + # is not part of the final stdenv. + (prevStage: + assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; + assert isFromBootstrapFiles prevStage."${localSystem.libc}"; + assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; + stageFun prevStage { + name = "bootstrap-stage-xgcc"; + overrides = final: prev: { + inherit (prevStage) ccWrapperStdenv coreutils gnugrep gettext bison texinfo zlib gnum4 perl; + patchelf = bootstrapTools; + ${localSystem.libc} = getLibc prevStage; + gmp = prev.gmp.override { cxx = false; }; + gcc-unwrapped = + (prev.gcc-unwrapped.override (commonGccOverrides // { + # The most logical name for this package would be something like + # "gcc-stage1". Unfortunately "stage" is already reserved for the + # layers of stdenv, so using "stage" in the name of this package + # would cause massive confusion. + # + # Gcc calls its "stage1" compiler `xgcc` (--disable-bootstrap results + # in `xgcc` being copied to $prefix/bin/gcc). So we imitate that. + # + name = "xgcc"; + + })).overrideAttrs (a: { + + # This signals to cc-wrapper (as overridden above in this file) to add `--sysroot` + # to `$out/nix-support/cc-cflags`. + passthru = a.passthru // { isXgcc = true; }; + + # Gcc will look for the C library headers in + # + # ${with_build_sysroot}${native_system_header_dir} + # + # The ordinary gcc expression sets `--with-build-sysroot=/` and sets + # `native-system-header-dir` to `"${lib.getDev stdenv.cc.libc}/include`. + # + # Unfortunately the value of "--with-native-system-header-dir=" gets "burned in" to the + # compiler, and it is quite difficult to get the compiler to change or ignore it + # afterwards. On the other hand, the `sysroot` is very easy to change; you can just pass + # a `--sysroot` flag to `gcc`. + # + # So we override the expression to remove the default settings for these flags, and + # replace them such that the concatenated value will be the same as before, but we split + # the value between the two variables differently: `--native-system-header-dir=/include`, + # and `--with-build-sysroot=${lib.getDev stdenv.cc.libc}`. + # + configureFlags = (a.configureFlags or []) ++ [ + "--with-native-system-header-dir=/include" + "--with-build-sysroot=${lib.getDev final.stdenv.cc.libc}" + ]; + + # This is a separate phase because gcc assembles its phase scripts + # in bash instead of nix (we should fix that). + preFixupPhases = (a.preFixupPhases or []) ++ [ "preFixupXgccPhase" ]; + + # This is needed to prevent "error: cycle detected in build of '...-xgcc-....drv' + # in the references of output 'lib' from output 'out'" + preFixupXgccPhase = '' + find $lib/lib/ -name \*.so\* -exec patchelf --shrink-rpath {} \; || true + ''; + }); + }; + }) # 2nd stdenv that contains our own rebuilt binutils and is used for # compiling our own Glibc. @@ -303,7 +389,7 @@ in # previous stage1 stdenv: assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; assert isFromBootstrapFiles prevStage."${localSystem.libc}"; - assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; assert isFromBootstrapFiles prevStage.coreutils; assert isFromBootstrapFiles prevStage.gnugrep; stageFun prevStage { @@ -313,7 +399,7 @@ in inherit (prevStage) ccWrapperStdenv gettext gcc-unwrapped coreutils gnugrep - perl gnum4 bison; + perl gnum4 bison texinfo which; dejagnu = super.dejagnu.overrideAttrs (a: { doCheck = false; } ); # We need libidn2 and its dependency libunistring as glibc dependency. @@ -378,11 +464,12 @@ in # binutils and rest of the bootstrap tools, including GCC. (prevStage: # previous stage2 stdenv: - assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isFromBootstrapFiles prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; assert isFromBootstrapFiles prevStage.coreutils; assert isFromBootstrapFiles prevStage.gnugrep; + assert lib.all isBuiltByNixpkgsCompiler (with prevStage; [ gmp isl_0_20 libmpc mpfr ]); stageFun prevStage { name = "bootstrap-stage3"; @@ -390,25 +477,20 @@ in inherit (prevStage) ccWrapperStdenv binutils coreutils gnugrep gettext - perl patchelf linuxHeaders gnum4 bison libidn2 libunistring; + perl patchelf linuxHeaders gnum4 bison libidn2 libunistring libxcrypt; + # We build a special copy of libgmp which doesn't use libstdc++, because + # xgcc++'s libstdc++ references the bootstrap-files (which is what + # compiles xgcc++). + gmp = super.gmp.override { cxx = false; }; + } // { ${localSystem.libc} = getLibc prevStage; - gcc-unwrapped = - let makeStaticLibrariesAndMark = pkg: - lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; }) - .overrideAttrs (a: { pname = "${a.pname}-stage3"; }); - in super.gcc-unwrapped.override { - # Link GCC statically against GMP etc. This makes sense because - # these builds of the libraries are only used by GCC, so it - # reduces the size of the stdenv closure. - gmp = makeStaticLibrariesAndMark super.gmp; - mpfr = makeStaticLibrariesAndMark super.mpfr; - libmpc = makeStaticLibrariesAndMark super.libmpc; - isl = makeStaticLibrariesAndMark super.isl_0_20; - # Use a deterministically built compiler - # see https://github.com/NixOS/nixpkgs/issues/108475 for context - reproducibleBuild = true; - profiledCompiler = false; - }; + gcc-unwrapped = (super.gcc-unwrapped.override (commonGccOverrides // { + inherit (prevStage) which; + } + )).overrideAttrs (a: { + # so we can add them to allowedRequisites below + passthru = a.passthru // { inherit (self) gmp mpfr libmpc isl; }; + }); }; extraNativeBuildInputs = [ prevStage.patchelf ] ++ # Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64. @@ -422,18 +504,11 @@ in # (prevStage: # previous stage3 stdenv: - assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isFromBootstrapFiles prevStage.coreutils; - assert isFromBootstrapFiles prevStage.gnugrep; - # Can assume prevStage.gcc-unwrapped has almost no code from - # bootstrapTools as gcc bootstraps internally. The only - # exceptions are crt files from glibc built bybootstrapTools - # used to link executables and libraries, and the - # bootstrapTools-built, statically-linked - # lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc - # (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d). + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isFromBootstrapFiles prevStage.coreutils; + assert isFromBootstrapFiles prevStage.gnugrep; stageFun prevStage { name = "bootstrap-stage4"; @@ -453,11 +528,6 @@ in }; }; - # force gmp to rebuild so we have the option of dynamically linking - # libgmp without creating a reference path from: - # stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap - gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; }); - # To allow users' overrides inhibit dependencies too heavy for # bootstrap, like guile: https://github.com/NixOS/nixpkgs/issues/181188 gnumake = super.gnumake.override { inBootstrap = true; }; @@ -494,11 +564,11 @@ in (prevStage: # previous stage4 stdenv; see stage3 comment regarding gcc, # which applies here as well. - assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isBuiltByNixpkgsCompiler prevStage.coreutils; - assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; { inherit config overlays; stdenv = import ../generic rec { @@ -546,11 +616,15 @@ in ) # More complicated cases ++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] ) - ++ [ /*propagated from .dev*/ linuxHeaders - binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params + ++ [ linuxHeaders # propagated from .dev + binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params gcc.cc.libgcc glibc.passthru.libgcc ] - ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") - [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ]; + ++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl") + [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ] + ++ (with gcc-unwrapped.passthru; [ + gmp libmpc mpfr isl + ]) + ; overrides = self: super: { inherit (prevStage) @@ -579,10 +653,10 @@ in (prevStage: # previous stage5 stdenv; see stage3 comment regarding gcc, # which applies here as well. - assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; - assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc}; - assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped; - assert isBuiltByNixpkgsCompiler prevStage.coreutils; - assert isBuiltByNixpkgsCompiler prevStage.gnugrep; + assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.${localSystem.libc}; + assert isBuiltByNixpkgsCompiler prevStage.gcc-unwrapped; + assert isBuiltByNixpkgsCompiler prevStage.coreutils; + assert isBuiltByNixpkgsCompiler prevStage.gnugrep; { inherit (prevStage) config overlays stdenv; }) ] diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index 3aa7f6a3df5..091130ebf93 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -2,6 +2,10 @@ let libc = pkgs.stdenv.cc.libc; + patchelf = pkgs.patchelf.overrideAttrs(previousAttrs: { + NIX_CFLAGS_COMPILE = (previousAttrs.NIX_CFLAGS_COMPILE or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + NIX_CFLAGS_LINK = (previousAttrs.NIX_CFLAGS_LINK or []) ++ [ "-static-libgcc" "-static-libstdc++" ]; + }); in with pkgs; rec { @@ -127,7 +131,7 @@ in with pkgs; rec { cp -d ${bootGCC.out}/bin/gcc $out/bin cp -d ${bootGCC.out}/bin/cpp $out/bin cp -d ${bootGCC.out}/bin/g++ $out/bin - cp -d ${bootGCC.lib}/lib/libgcc_s.so* $out/lib + cp ${bootGCC.lib}/lib/libgcc_s.so* $out/lib cp -d ${bootGCC.lib}/lib/libstdc++.so* $out/lib cp -d ${bootGCC.out}/lib/libssp.a* $out/lib cp -d ${bootGCC.out}/lib/libssp_nonshared.a $out/lib @@ -149,6 +153,7 @@ in with pkgs; rec { rm -rf $out/include/c++/*/ext/parallel cp -d ${gmpxx.out}/lib/libgmp*.so* $out/lib + cp -d ${isl.out}/lib/libisl*.so* $out/lib cp -d ${mpfr.out}/lib/libmpfr*.so* $out/lib cp -d ${libmpc.out}/lib/libmpc*.so* $out/lib cp -d ${zlib.out}/lib/libz.so* $out/lib |