diff options
Diffstat (limited to 'pkgs/stdenv')
23 files changed, 1454 insertions, 653 deletions
diff --git a/pkgs/stdenv/adapters.nix b/pkgs/stdenv/adapters.nix index 8b23d3dadd2..a8e984d6174 100644 --- a/pkgs/stdenv/adapters.nix +++ b/pkgs/stdenv/adapters.nix @@ -15,7 +15,7 @@ rec { # Used to override packages in stdenv like Make. Should not be used # for other dependencies. overrideInStdenv = stdenv: pkgs: - stdenv.override (prev: { allowedRequisites = null; extraBuildInputs = prev.extraBuildInputs or [] ++ pkgs; }); + stdenv.override (prev: { allowedRequisites = null; extraBuildInputs = (prev.extraBuildInputs or []) ++ pkgs; }); # Override the setup script of stdenv. Useful for testing new @@ -34,7 +34,7 @@ rec { makeStaticBinaries = stdenv: let stdenv' = if stdenv.hostPlatform.libc != "glibc" then stdenv else stdenv.override (prev: { - extraBuildInputs = prev.extraBuildInputs or [] ++ [ + extraBuildInputs = (prev.extraBuildInputs or []) ++ [ stdenv.glibc.static ]; }); @@ -44,6 +44,7 @@ rec { then throw "Cannot build fully static binaries on Darwin/macOS" else stdenv'.mkDerivation (args // { NIX_CFLAGS_LINK = toString (args.NIX_CFLAGS_LINK or "") + " -static"; + } // pkgs.lib.optionalAttrs (!(args.dontAddStaticConfigureFlags or false)) { configureFlags = (args.configureFlags or []) ++ [ "--disable-shared" # brrr... ]; @@ -56,6 +57,7 @@ rec { makeStaticLibraries = stdenv: stdenv // { mkDerivation = args: stdenv.mkDerivation (args // { dontDisableStatic = true; + } // pkgs.lib.optionalAttrs (!(args.dontAddStaticConfigureFlags or false)) { configureFlags = (args.configureFlags or []) ++ [ "--enable-static" "--disable-shared" @@ -110,7 +112,7 @@ rec { */ replaceMaintainersField = stdenv: pkgs: maintainers: stdenv // { mkDerivation = args: - stdenv.lib.recursiveUpdate + pkgs.lib.recursiveUpdate (stdenv.mkDerivation args) { meta.maintainers = maintainers; }; }; diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix index 6ac03b7908f..18a6e2a4737 100644 --- a/pkgs/stdenv/cross/default.nix +++ b/pkgs/stdenv/cross/default.nix @@ -48,7 +48,9 @@ in lib.init bootStages ++ [ # Prior overrides are surely not valid as packages built with this run on # a different platform, and so are disabled. overrides = _: _: {}; - extraBuildInputs = [ ]; # Old ones run on wrong platform + extraBuildInputs = [ ] # Old ones run on wrong platform + ++ lib.optionals hostPlatform.isDarwin [ buildPackages.targetPackages.darwin.apple_sdk.frameworks.CoreFoundation ] + ; allowedRequisites = null; hasCC = !targetPlatform.isGhcjs; @@ -63,8 +65,10 @@ in lib.init bootStages ++ [ # `tryEval` wouldn't catch, wrecking accessing previous stages # when there is a C compiler and everything should be fine. then throw "no C compiler provided for this platform" + else if crossSystem.isDarwin + then buildPackages.llvmPackages.clang else if crossSystem.useLLVM or false - then buildPackages.llvmPackages_8.lldClang + then buildPackages.llvmPackages.clangUseLLVM else buildPackages.gcc; extraNativeBuildInputs = old.extraNativeBuildInputs diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index 9de6ef63bfe..b5a9f0cc5cb 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -1,42 +1,66 @@ { lib -, localSystem, crossSystem, config, overlays, crossOverlays ? [] -# The version of darwin.apple_sdk used for sources provided by apple. -, appleSdkVersion ? "10.12" -# Minimum required macOS version, used both for compatibility as well as reproducability. -, macosVersionMin ? "10.12" -# Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools -, bootstrapFiles ? let - fetch = { file, sha256, executable ? true }: import <nix/fetchurl.nix> { - url = "http://tarballs.nixos.org/stdenv-darwin/x86_64/d5bdfcbfe6346761a332918a267e82799ec954d2/${file}"; - inherit (localSystem) system; - inherit sha256 executable; - }; in { - sh = fetch { file = "sh"; sha256 = "07wm33f1yzfpcd3rh42f8g096k4cvv7g65p968j28agzmm2s7s8m"; }; - bzip2 = fetch { file = "bzip2"; sha256 = "0y9ri2aprkrp2dkzm6229l0mw4rxr2jy7vvh3d8mxv2698v2kdbm"; }; - mkdir = fetch { file = "mkdir"; sha256 = "0sb07xpy66ws6f2jfnpjibyimzb71al8n8c6y4nr8h50al3g90nr"; }; - cpio = fetch { file = "cpio"; sha256 = "0r5c54hg678w7zydx27bzl9p3v9fs25y5ix6vdfi1ilqim7xh65n"; }; - tarball = fetch { file = "bootstrap-tools.cpio.bz2"; sha256 = "18hp5w6klr8g307ap4368r255qpzg9r0vwg9vqvj8f2zy1xilcjf"; executable = false; }; - } +, localSystem +, crossSystem +, config +, overlays +, crossOverlays ? [ ] +, bootstrapLlvmVersion ? if localSystem.isAarch64 then "11.1.0" else "7.1.0" + # Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools +, bootstrapFiles ? if localSystem.isAarch64 then + let + fetch = { file, sha256, executable ? true }: import <nix/fetchurl.nix> { + url = "http://tarballs.nixos.org/stdenv-darwin/aarch64/20acd4c4f14040485f40e55c0a76c186aa8ca4f3/${file}"; + inherit (localSystem) system; + inherit sha256 executable; + }; in + { + sh = fetch { file = "sh"; sha256 = "17m3xrlbl99j3vm7rzz3ghb47094dyddrbvs2a6jalczvmx7spnj"; }; + bzip2 = fetch { file = "bzip2"; sha256 = "1khs8s5klf76plhlvlc1ma838r8pc1qigk9f5bdycwgbn0nx240q"; }; + mkdir = fetch { file = "mkdir"; sha256 = "1m9nk90paazl93v43myv2ay68c1arz39pqr7lk5ddbgb177hgg8a"; }; + cpio = fetch { file = "cpio"; sha256 = "17pxq61yjjvyd738fy9f392hc9cfzkl612sdr9rxr3v0dgvm8y09"; }; + tarball = fetch { file = "bootstrap-tools.cpio.bz2"; sha256 = "1v2332k33akm6mrm4bj749rxnnmc2pkbgcslmd0bbkf76bz2ildy"; executable = false; }; + } + else + let + fetch = { file, sha256, executable ? true }: import <nix/fetchurl.nix> { + url = "http://tarballs.nixos.org/stdenv-darwin/x86_64/05ef940b94fe76e7ac06ea45a625adc8e4be96f9/${file}"; + inherit (localSystem) system; + inherit sha256 executable; + }; in + { + sh = fetch { file = "sh"; sha256 = "sha256-igMAVEfumFv/LUNTGfNi2nSehgTNIP4Sg+f3L7u6SMA="; }; + bzip2 = fetch { file = "bzip2"; sha256 = "sha256-K3rhkJZipudT1Jgh+l41Y/fNsMkrPtiAsNRDha/lpZI="; }; + mkdir = fetch { file = "mkdir"; sha256 = "sha256-VddFELwLDJGNADKB1fWwWPBtIAlEUgJv2hXRmC4NEeM="; }; + cpio = fetch { file = "cpio"; sha256 = "sha256-SWkwvLaFyV44kLKL2nx720SvcL4ej/p2V/bX3uqAGO0="; }; + tarball = fetch { file = "bootstrap-tools.cpio.bz2"; sha256 = "sha256-b65dXbIm6o6s6U8tAiGpR6SMfvfn/VFcZgTHBetJZis="; executable = false; }; + } }: assert crossSystem == localSystem; let - inherit (localSystem) system platform; + inherit (localSystem) system; + + useAppleSDKLibs = localSystem.isAarch64; + haveKRB5 = localSystem.isx86_64; + + # final toolchain is injected into llvmPackages_${finalLlvmVersion} + finalLlvmVersion = lib.versions.major bootstrapLlvmVersion; + finalLlvmPackages = "llvmPackages_${finalLlvmVersion}"; commonImpureHostDeps = [ "/bin/sh" "/usr/lib/libSystem.B.dylib" "/usr/lib/system/libunc.dylib" # This dependency is "hidden", so our scanning code doesn't pick it up ]; -in rec { + +in +rec { commonPreHook = '' export NIX_ENFORCE_NO_NATIVE=''${NIX_ENFORCE_NO_NATIVE-1} export NIX_ENFORCE_PURITY=''${NIX_ENFORCE_PURITY-1} export NIX_IGNORE_LD_THROUGH_GCC=1 - export SDKROOT= - - export MACOSX_DEPLOYMENT_TARGET=${macosVersionMin} + unset SDKROOT # Workaround for https://openradar.appspot.com/22671534 on 10.11. export gl_cv_func_getcwd_abort_bug=no @@ -44,27 +68,30 @@ in rec { stripAllFlags=" " # the Darwin "strip" command doesn't know "-s" ''; - bootstrapTools = derivation { + bootstrapTools = derivation ({ inherit system; - name = "bootstrap-tools"; + name = "bootstrap-tools"; builder = bootstrapFiles.sh; # Not a filename! Attribute 'sh' on bootstrapFiles - args = [ ./unpack-bootstrap-tools.sh ]; + args = if localSystem.isAarch64 then [ ./unpack-bootstrap-tools-aarch64.sh ] else [ ./unpack-bootstrap-tools.sh ]; inherit (bootstrapFiles) mkdir bzip2 cpio tarball; - reexportedLibrariesFile = - ../../os-specific/darwin/apple-source-releases/Libsystem/reexported_libraries; __impureHostDeps = commonImpureHostDeps; - }; - - stageFun = step: last: {shell ? "${bootstrapTools}/bin/bash", - overrides ? (self: super: {}), - extraPreHook ? "", - extraNativeBuildInputs, - extraBuildInputs, - libcxx, - allowedRequisites ? null}: + } // lib.optionalAttrs (config.contentAddressedByDefault or false) { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }); + + stageFun = step: last: { shell ? "${bootstrapTools}/bin/bash" + , overrides ? (self: super: { }) + , extraPreHook ? "" + , extraNativeBuildInputs + , extraBuildInputs + , libcxx + , allowedRequisites ? null + }: let name = "bootstrap-stage${toString step}"; @@ -72,40 +99,75 @@ in rec { inherit (last) stdenv; }; - coreutils = { name = "${name}-coreutils"; outPath = bootstrapTools; }; - gnugrep = { name = "${name}-gnugrep"; outPath = bootstrapTools; }; - - bintools = import ../../build-support/bintools-wrapper { - inherit shell; - inherit (last) stdenvNoCC; - - nativeTools = false; - nativeLibc = false; - inherit buildPackages coreutils gnugrep; - libc = last.pkgs.darwin.Libsystem; - bintools = { name = "${name}-binutils"; outPath = bootstrapTools; }; - }; - - cc = if last == null then "/dev/null" else import ../../build-support/cc-wrapper { - inherit shell; - inherit (last) stdenvNoCC; - - extraPackages = []; + doSign = localSystem.isAarch64 && last != null; + doUpdateAutoTools = localSystem.isAarch64 && last != null; + + mkExtraBuildCommands = cc: '' + rsrc="$out/resource-root" + mkdir "$rsrc" + ln -s "${cc.lib or cc}/lib/clang/${cc.version}/include" "$rsrc" + ln -s "${last.pkgs."${finalLlvmPackages}".compiler-rt.out}/lib" "$rsrc/lib" + echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags + ''; + + mkCC = overrides: import ../../build-support/cc-wrapper ( + let args = { + inherit lib shell; + inherit (last) stdenvNoCC; + + nativeTools = false; + nativeLibc = false; + inherit buildPackages libcxx; + inherit (last.pkgs) coreutils gnugrep; + bintools = last.pkgs.darwin.binutils; + libc = last.pkgs.darwin.Libsystem; + isClang = true; + cc = last.pkgs."${finalLlvmPackages}".clang-unwrapped; + }; in args // (overrides args) + ); + + cc = if last == null then "/dev/null" else + mkCC ({ cc, ... }: { + extraPackages = [ + last.pkgs."${finalLlvmPackages}".libcxxabi + last.pkgs."${finalLlvmPackages}".compiler-rt + ]; + extraBuildCommands = mkExtraBuildCommands cc; + }); - nativeTools = false; - nativeLibc = false; - inherit buildPackages coreutils gnugrep bintools libcxx; - libc = last.pkgs.darwin.Libsystem; - isClang = true; - cc = { name = "${name}-clang"; outPath = bootstrapTools; }; - }; + ccNoLibcxx = if last == null then "/dev/null" else + mkCC ({ cc, ... }: { + libcxx = null; + extraPackages = [ + last.pkgs."${finalLlvmPackages}".compiler-rt + ]; + extraBuildCommands = '' + echo "-rtlib=compiler-rt" >> $out/nix-support/cc-cflags + echo "-B${last.pkgs."${finalLlvmPackages}".compiler-rt}/lib" >> $out/nix-support/cc-cflags + echo "-nostdlib++" >> $out/nix-support/cc-cflags + '' + mkExtraBuildCommands cc; + }); thisStdenv = import ../generic { name = "${name}-stdenv-darwin"; - inherit config shell extraNativeBuildInputs extraBuildInputs; + inherit config shell extraBuildInputs; + + extraNativeBuildInputs = extraNativeBuildInputs ++ lib.optionals doUpdateAutoTools [ + last.pkgs.updateAutotoolsGnuConfigScriptsHook + last.pkgs.gnu-config + ]; + allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [ - cc.expand-response-params cc.bintools + cc.expand-response-params + cc.bintools + ] ++ lib.optionals doUpdateAutoTools [ + last.pkgs.updateAutotoolsGnuConfigScriptsHook + last.pkgs.gnu-config + ] ++ lib.optionals doSign [ + last.pkgs.darwin.postLinkSignHook + last.pkgs.darwin.sigtool + last.pkgs.darwin.signingUtils ]; buildPlatform = localSystem; @@ -122,7 +184,7 @@ in rec { ${commonPreHook} ${extraPreHook} ''; - initialPath = [ bootstrapTools ]; + initialPath = [ bootstrapTools ]; fetchurlBoot = import ../../build-support/fetchurl { inherit lib; @@ -134,32 +196,164 @@ in rec { __stdenvImpureHostDeps = commonImpureHostDeps; __extraImpureHostDeps = commonImpureHostDeps; - extraAttrs = { - inherit macosVersionMin appleSdkVersion platform; + overrides = self: super: (overrides self super) // { + inherit ccNoLibcxx; + fetchurl = thisStdenv.fetchurlBoot; }; - overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; - in { + in + { inherit config overlays; stdenv = thisStdenv; }; stage0 = stageFun 0 null { overrides = self: super: with stage0; { - darwin = super.darwin // { + coreutils = stdenv.mkDerivation { + name = "bootstrap-stage0-coreutils"; + buildCommand = '' + mkdir -p $out + ln -s ${bootstrapTools}/bin $out/bin + ''; + }; + + gnugrep = stdenv.mkDerivation { + name = "bootstrap-stage0-gnugrep"; + buildCommand = '' + mkdir -p $out + ln -s ${bootstrapTools}/bin $out/bin + ''; + }; + + pbzx = stdenv.mkDerivation { + name = "bootstrap-stage0-pbzx"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + ln -s ${bootstrapTools}/bin/pbzx $out/bin + ''; + }; + + cpio = stdenv.mkDerivation { + name = "bootstrap-stage0-cpio"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + ln -s ${bootstrapFiles.cpio} $out/bin/cpio + ''; + }; + + darwin = super.darwin.overrideScope (selfDarwin: superDarwin: { + darwin-stubs = superDarwin.darwin-stubs.override { inherit (self) stdenvNoCC fetchurl; }; + + dyld = { + name = "bootstrap-stage0-dyld"; + buildCommand = '' + mkdir -p $out + ln -s ${bootstrapTools}/lib $out/lib + ln -s ${bootstrapTools}/include $out/include + ''; + }; + + sigtool = stdenv.mkDerivation { + name = "bootstrap-stage0-sigtool"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + ln -s ${bootstrapTools}/bin/sigtool $out/bin + + # Rewrite nuked references + sed -e "s|[^( ]*\bsigtool\b|$out/bin/sigtool|g" \ + ${bootstrapTools}/bin/codesign > $out/bin/codesign + chmod a+x $out/bin/codesign + ''; + }; + + print-reexports = stdenv.mkDerivation { + name = "bootstrap-stage0-print-reexports"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + ln -s ${bootstrapTools}/bin/print-reexports $out/bin + ''; + }; + + rewrite-tbd = stdenv.mkDerivation { + name = "bootstrap-stage0-rewrite-tbd"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/bin + ln -s ${bootstrapTools}/bin/rewrite-tbd $out/bin + ''; + }; + + binutils-unwrapped = { name = "bootstrap-stage0-binutils"; outPath = bootstrapTools; }; + + cctools = { + name = "bootstrap-stage0-cctools"; + outPath = bootstrapTools; + targetPrefix = ""; + }; + + binutils = lib.makeOverridable (import ../../build-support/bintools-wrapper) { + shell = "${bootstrapTools}/bin/bash"; + inherit lib; + inherit (self) stdenvNoCC; + + nativeTools = false; + nativeLibc = false; + inherit (self) buildPackages coreutils gnugrep; + libc = selfDarwin.Libsystem; + bintools = selfDarwin.binutils-unwrapped; + inherit (selfDarwin) postLinkSignHook signingUtils; + }; + } // lib.optionalAttrs (! useAppleSDKLibs) { + CF = stdenv.mkDerivation { + name = "bootstrap-stage0-CF"; + buildCommand = '' + mkdir -p $out/Library/Frameworks + ln -s ${bootstrapTools}/Library/Frameworks/CoreFoundation.framework $out/Library/Frameworks + ''; + }; + Libsystem = stdenv.mkDerivation { name = "bootstrap-stage0-Libsystem"; buildCommand = '' mkdir -p $out - ln -s ${bootstrapTools}/lib $out/lib + + cp -r ${selfDarwin.darwin-stubs}/usr/lib $out/lib + chmod -R +w $out/lib + substituteInPlace $out/lib/libSystem.B.tbd --replace /usr/lib/system $out/lib/system + + ln -s libSystem.B.tbd $out/lib/libSystem.tbd + + for name in c dbm dl info m mx poll proc pthread rpcsvc util gcc_s.10.4 gcc_s.10.5; do + ln -s libSystem.tbd $out/lib/lib$name.tbd + done + + ln -s ${bootstrapTools}/lib/*.o $out/lib + + ln -s ${bootstrapTools}/lib/libresolv.9.dylib $out/lib + ln -s libresolv.9.dylib $out/lib/libresolv.dylib + ln -s ${bootstrapTools}/include-Libsystem $out/include ''; }; - dyld = bootstrapTools; - }; + }); + + "${finalLlvmPackages}" = { + clang-unwrapped = stdenv.mkDerivation { + name = "bootstrap-stage0-clang"; + version = bootstrapLlvmVersion; + buildCommand = '' + mkdir -p $out/lib + ln -s ${bootstrapTools}/bin $out/bin + ln -s ${bootstrapTools}/lib/clang $out/lib/clang + ln -s ${bootstrapTools}/include $out/include + ''; + }; - llvmPackages_7 = { libcxx = stdenv.mkDerivation { name = "bootstrap-stage0-libcxx"; phases = [ "installPhase" "fixupPhase" ]; @@ -180,259 +374,431 @@ in rec { ln -s ${bootstrapTools}/lib/libc++abi.dylib $out/lib/libc++abi.dylib ''; }; + + compiler-rt = stdenv.mkDerivation { + name = "bootstrap-stage0-compiler-rt"; + buildCommand = '' + mkdir -p $out/lib + ln -s ${bootstrapTools}/lib/libclang_rt* $out/lib + ln -s ${bootstrapTools}/lib/darwin $out/lib/darwin + ''; + }; }; }; - extraNativeBuildInputs = []; - extraBuildInputs = []; + extraNativeBuildInputs = [ ]; + extraBuildInputs = [ ]; libcxx = null; }; - stage1 = prevStage: let - persistent = self: super: with prevStage; { - cmake = super.cmake.override { - isBootstrap = true; - useSharedLibraries = false; - }; + stage1 = prevStage: + let + persistent = self: super: with prevStage; { + cmake = super.cmakeMinimal; - python3 = super.python3Minimal; + inherit pbzx cpio; - ninja = super.ninja.override { buildDocs = false; }; + python3 = super.python3Minimal; - darwin = super.darwin // { - cctools = super.darwin.cctools.override { - enableTapiSupport = false; - }; - }; - }; - in with prevStage; stageFun 1 prevStage { - extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\""; - extraNativeBuildInputs = []; - extraBuildInputs = [ ]; - libcxx = pkgs.libcxx; + ninja = super.ninja.override { buildDocs = false; }; - allowedRequisites = - [ bootstrapTools ] ++ (with pkgs; [ libcxx libcxxabi ]) ++ [ pkgs.darwin.Libsystem ]; + "${finalLlvmPackages}" = super."${finalLlvmPackages}" // ( + let + tools = super."${finalLlvmPackages}".tools.extend (_: _: { + inherit (pkgs."${finalLlvmPackages}") clang-unwrapped; + }); + libraries = super."${finalLlvmPackages}".libraries.extend (_: _: { + inherit (pkgs."${finalLlvmPackages}") compiler-rt libcxx libcxxabi; + }); + in + { inherit tools libraries; } // tools // libraries + ); - overrides = persistent; - }; + darwin = super.darwin.overrideScope (selfDarwin: _: { + inherit (darwin) rewrite-tbd binutils-unwrapped; - stage2 = prevStage: let - persistent = self: super: with prevStage; { - inherit - zlib patchutils m4 scons flex perl bison unifdef unzip openssl python3 - libxml2 gettext sharutils gmp libarchive ncurses pkg-config libedit groff - openssh sqlite sed serf openldap db cyrus-sasl expat apr-util subversion xz - findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils - libssh2 nghttp2 libkrb5 ninja; - - darwin = super.darwin // { - inherit (darwin) - dyld Libsystem xnu configd ICU libdispatch libclosure launchd CF; + signingUtils = darwin.signingUtils.override { + inherit (selfDarwin) sigtool; + }; + + binutils = darwin.binutils.override { + coreutils = self.coreutils; + libc = selfDarwin.Libsystem; + inherit (selfDarwin) postLinkSignHook signingUtils; + }; + }); }; + in + with prevStage; stageFun 1 prevStage { + extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\""; + extraNativeBuildInputs = [ ]; + extraBuildInputs = [ pkgs.darwin.CF ]; + libcxx = pkgs."${finalLlvmPackages}".libcxx; + + allowedRequisites = + [ bootstrapTools ] ++ + (with pkgs; [ coreutils gnugrep ]) ++ + (with pkgs."${finalLlvmPackages}"; [ libcxx libcxxabi compiler-rt clang-unwrapped ]) ++ + (with pkgs.darwin; [ Libsystem CF ] ++ lib.optional useAppleSDKLibs objc4); + + overrides = persistent; }; - in with prevStage; stageFun 2 prevStage { - extraPreHook = '' - export PATH_LOCALE=${pkgs.darwin.locale}/share/locale - ''; - - extraNativeBuildInputs = [ pkgs.xz ]; - extraBuildInputs = [ pkgs.darwin.CF ]; - libcxx = pkgs.libcxx; - - allowedRequisites = - [ bootstrapTools ] ++ - (with pkgs; [ - xz.bin xz.out libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out - nghttp2.lib libkrb5 - ]) ++ - (with pkgs.darwin; [ dyld Libsystem CF ICU locale ]); - - overrides = persistent; - }; - stage3 = prevStage: let - persistent = self: super: with prevStage; { - inherit - patchutils m4 scons flex perl bison unifdef unzip openssl python3 - gettext sharutils libarchive pkg-config groff bash subversion - openssh sqlite sed serf openldap db cyrus-sasl expat apr-util - findfreetype libssh curl cmake autoconf automake libtool cpio - libssh2 nghttp2 libkrb5 ninja; - - # Avoid pulling in a full python and its extra dependencies for the llvm/clang builds. - libxml2 = super.libxml2.override { pythonSupport = false; }; - - llvmPackages_7 = super.llvmPackages_7 // (let - libraries = super.llvmPackages_7.libraries.extend (_: _: { - inherit (llvmPackages_7) libcxx libcxxabi; + stage2 = prevStage: + let + persistent = self: super: with prevStage; { + inherit + zlib patchutils m4 scons flex perl bison unifdef unzip openssl python3 + libxml2 gettext sharutils gmp libarchive ncurses pkg-config libedit groff + openssh sqlite sed serf openldap db cyrus-sasl expat apr-util subversion xz + findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils + libssh2 nghttp2 libkrb5 ninja brotli; + + "${finalLlvmPackages}" = super."${finalLlvmPackages}" // ( + let + tools = super."${finalLlvmPackages}".tools.extend (_: _: { + inherit (pkgs."${finalLlvmPackages}") clang-unwrapped; + }); + libraries = super."${finalLlvmPackages}".libraries.extend (_: libSuper: { + inherit (pkgs."${finalLlvmPackages}") compiler-rt; + libcxx = libSuper.libcxx.override { + stdenv = overrideCC self.stdenv self.ccNoLibcxx; + }; + libcxxabi = libSuper.libcxxabi.override ({ + stdenv = overrideCC self.stdenv self.ccNoLibcxx; + } // lib.optionalAttrs (finalLlvmVersion == "7") { + # TODO: the bootstrapping of llvm packages isn't consistent. + # `standalone` may be redundant if darwin behaves like useLLVM (or + # has useLLVM = true). + standalone = true; + }); + }); + in + { inherit tools libraries; } // tools // libraries + ); + + darwin = super.darwin.overrideScope (_: _: { + inherit (darwin) + binutils dyld Libsystem xnu configd ICU libdispatch libclosure + launchd CF objc4 darwin-stubs sigtool postLinkSignHook signingUtils; }); - in { inherit libraries; } // libraries); - - darwin = super.darwin // { - inherit (darwin) - dyld Libsystem xnu configd libdispatch libclosure launchd libiconv locale; }; + in + with prevStage; stageFun 2 prevStage { + extraPreHook = '' + export PATH_LOCALE=${pkgs.darwin.locale}/share/locale + ''; + + extraNativeBuildInputs = [ pkgs.xz ]; + extraBuildInputs = [ pkgs.darwin.CF ]; + libcxx = pkgs."${finalLlvmPackages}".libcxx; + + allowedRequisites = + [ bootstrapTools ] ++ + (with pkgs; [ + xz.bin + xz.out + zlib + libxml2.out + curl.out + openssl.out + libssh2.out + nghttp2.lib + coreutils + gnugrep + pcre.out + gmp + libiconv + brotli.lib + ] ++ lib.optional haveKRB5 libkrb5) ++ + (with pkgs."${finalLlvmPackages}"; [ + libcxx + libcxxabi + compiler-rt + clang-unwrapped + ]) ++ + (with pkgs.darwin; [ dyld Libsystem CF ICU locale ] ++ lib.optional useAppleSDKLibs objc4); + + overrides = persistent; }; - in with prevStage; stageFun 3 prevStage { - shell = "${pkgs.bash}/bin/bash"; - - # We have a valid shell here (this one has no bootstrap-tools runtime deps) so stageFun - # enables patchShebangs above. Unfortunately, patchShebangs ignores our $SHELL setting - # and instead goes by $PATH, which happens to contain bootstrapTools. So it goes and - # patches our shebangs back to point at bootstrapTools. This makes sure bash comes first. - extraNativeBuildInputs = with pkgs; [ xz ]; - extraBuildInputs = [ pkgs.darwin.CF pkgs.bash ]; - libcxx = pkgs.libcxx; - - extraPreHook = '' - export PATH=${pkgs.bash}/bin:$PATH - export PATH_LOCALE=${pkgs.darwin.locale}/share/locale - ''; - - allowedRequisites = - [ bootstrapTools ] ++ - (with pkgs; [ - xz.bin xz.out bash libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out - nghttp2.lib libkrb5 - ]) ++ - (with pkgs.darwin; [ dyld ICU Libsystem locale ]); - - overrides = persistent; - }; - stage4 = prevStage: let - persistent = self: super: with prevStage; { - inherit - gnumake gzip gnused bzip2 gawk ed xz patch bash python3 - ncurses libffi zlib gmp pcre gnugrep - coreutils findutils diffutils patchutils ninja libxml2; - - # Hack to make sure we don't link ncurses in bootstrap tools. The proper - # solution is to avoid passing -L/nix-store/...-bootstrap-tools/lib, - # quite a sledgehammer just to get the C runtime. - gettext = super.gettext.overrideAttrs (drv: { - configureFlags = drv.configureFlags ++ [ - "--disable-curses" - ]; - }); - - llvmPackages_7 = super.llvmPackages_7 // (let - tools = super.llvmPackages_7.tools.extend (llvmSelf: _: { - clang-unwrapped = llvmPackages_7.clang-unwrapped.override { llvm = llvmSelf.llvm; }; - llvm = llvmPackages_7.llvm.override { inherit libxml2; }; - }); - libraries = super.llvmPackages_7.libraries.extend (llvmSelf: _: { - inherit (llvmPackages_7) libcxx libcxxabi compiler-rt; + stage3 = prevStage: + let + persistent = self: super: with prevStage; { + inherit + patchutils m4 scons flex perl bison unifdef unzip openssl python3 + gettext sharutils libarchive pkg-config groff bash subversion + openssh sqlite sed serf openldap db cyrus-sasl expat apr-util + findfreetype libssh curl cmake autoconf automake libtool cpio + libssh2 nghttp2 libkrb5 ninja; + + # Avoid pulling in a full python and its extra dependencies for the llvm/clang builds. + libxml2 = super.libxml2.override { pythonSupport = false; }; + + "${finalLlvmPackages}" = super."${finalLlvmPackages}" // ( + let + libraries = super."${finalLlvmPackages}".libraries.extend (_: _: { + inherit (pkgs."${finalLlvmPackages}") libcxx libcxxabi; + }); + in + { inherit libraries; } // libraries + ); + + darwin = super.darwin.overrideScope (_: _: { + inherit (darwin) + dyld Libsystem xnu configd libdispatch libclosure launchd libiconv + locale darwin-stubs sigtool; }); - in { inherit tools libraries; } // tools // libraries); - - darwin = super.darwin // rec { - inherit (darwin) dyld Libsystem libiconv locale; - - cctools = super.darwin.cctools.override { enableTapiSupport = false; }; - CF = super.darwin.CF.override { - inherit libxml2; - python3 = prevStage.python3; - }; }; + in + with prevStage; stageFun 3 prevStage { + shell = "${pkgs.bash}/bin/bash"; + + # We have a valid shell here (this one has no bootstrap-tools runtime deps) so stageFun + # enables patchShebangs above. Unfortunately, patchShebangs ignores our $SHELL setting + # and instead goes by $PATH, which happens to contain bootstrapTools. So it goes and + # patches our shebangs back to point at bootstrapTools. This makes sure bash comes first. + extraNativeBuildInputs = with pkgs; [ xz ]; + extraBuildInputs = [ pkgs.darwin.CF pkgs.bash ]; + libcxx = pkgs."${finalLlvmPackages}".libcxx; + + extraPreHook = '' + export PATH=${pkgs.bash}/bin:$PATH + export PATH_LOCALE=${pkgs.darwin.locale}/share/locale + ''; + + allowedRequisites = + [ bootstrapTools ] ++ + (with pkgs; [ + xz.bin + xz.out + bash + zlib + libxml2.out + curl.out + openssl.out + libssh2.out + nghttp2.lib + coreutils + gnugrep + pcre.out + gmp + libiconv + brotli.lib + ] ++ lib.optional haveKRB5 libkrb5) ++ + (with pkgs."${finalLlvmPackages}"; [ + libcxx + libcxx.dev + libcxxabi + libcxxabi.dev + compiler-rt + clang-unwrapped + ]) ++ + (with pkgs.darwin; [ dyld ICU Libsystem locale ] ++ lib.optional useAppleSDKLibs objc4); + + overrides = persistent; }; - in with prevStage; stageFun 4 prevStage { - shell = "${pkgs.bash}/bin/bash"; - extraNativeBuildInputs = with pkgs; [ xz ]; - extraBuildInputs = [ pkgs.darwin.CF pkgs.bash ]; - libcxx = pkgs.libcxx; - - extraPreHook = '' - export PATH_LOCALE=${pkgs.darwin.locale}/share/locale - ''; - overrides = persistent; - }; - stdenvDarwin = prevStage: let - pkgs = prevStage; - persistent = self: super: with prevStage; { - inherit - gnumake gzip gnused bzip2 gawk ed xz patch bash - ncurses libffi zlib llvm gmp pcre gnugrep - coreutils findutils diffutils patchutils; - - llvmPackages_7 = super.llvmPackages_7 // (let - tools = super.llvmPackages_7.tools.extend (_: super: { - inherit (llvmPackages_7) llvm clang-unwrapped; + stage4 = prevStage: + let + persistent = self: super: with prevStage; { + inherit + gnumake gzip gnused bzip2 gawk ed xz patch bash python3 + ncurses libffi zlib gmp pcre gnugrep cmake + coreutils findutils diffutils patchutils ninja libxml2; + + # Hack to make sure we don't link ncurses in bootstrap tools. The proper + # solution is to avoid passing -L/nix-store/...-bootstrap-tools/lib, + # quite a sledgehammer just to get the C runtime. + gettext = super.gettext.overrideAttrs (drv: { + configureFlags = drv.configureFlags ++ [ + "--disable-curses" + ]; }); - libraries = super.llvmPackages_7.libraries.extend (_: _: { - inherit (llvmPackages_7) compiler-rt libcxx libcxxabi; + + "${finalLlvmPackages}" = super."${finalLlvmPackages}" // ( + let + tools = super."${finalLlvmPackages}".tools.extend (llvmSelf: _: { + clang-unwrapped-all-outputs = pkgs."${finalLlvmPackages}".clang-unwrapped-all-outputs.override { llvm = llvmSelf.llvm; }; + libllvm = pkgs."${finalLlvmPackages}".libllvm.override { inherit libxml2; }; + }); + libraries = super."${finalLlvmPackages}".libraries.extend (llvmSelf: _: { + inherit (pkgs."${finalLlvmPackages}") libcxx libcxxabi compiler-rt; + }); + in + { inherit tools libraries; } // tools // libraries + ); + + darwin = super.darwin.overrideScope (_: superDarwin: { + inherit (darwin) dyld Libsystem libiconv locale darwin-stubs; + + # See useAppleSDKLibs in darwin-packages.nix + CF = if useAppleSDKLibs then super.darwin.CF else + superDarwin.CF.override { + inherit libxml2; + python3 = prevStage.python3; + }; }); - in { inherit tools libraries; } // tools // libraries); + }; + in + with prevStage; stageFun 4 prevStage { + shell = "${pkgs.bash}/bin/bash"; + extraNativeBuildInputs = with pkgs; [ xz ]; + extraBuildInputs = [ pkgs.darwin.CF pkgs.bash ]; + libcxx = pkgs."${finalLlvmPackages}".libcxx; + + extraPreHook = '' + export PATH_LOCALE=${pkgs.darwin.locale}/share/locale + ''; + overrides = persistent; + }; - darwin = super.darwin // { - inherit (darwin) dyld ICU Libsystem libiconv; + stdenvDarwin = prevStage: + let + doSign = localSystem.isAarch64; + pkgs = prevStage; + persistent = self: super: with prevStage; { + inherit + gnumake gzip gnused bzip2 gawk ed xz patch bash + ncurses libffi zlib gmp pcre gnugrep + coreutils findutils diffutils patchutils pbzx; + + darwin = super.darwin.overrideScope (_: _: { + inherit (darwin) dyld ICU Libsystem Csu libiconv rewrite-tbd; + } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) { + inherit (darwin) binutils binutils-unwrapped cctools; + }); } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) { - inherit (darwin) binutils binutils-unwrapped cctools; + inherit llvm; + + # Need to get rid of these when cross-compiling. + "${finalLlvmPackages}" = super."${finalLlvmPackages}" // ( + let + tools = super."${finalLlvmPackages}".tools.extend (_: super: { + inherit (pkgs."${finalLlvmPackages}") llvm clang-unwrapped; + }); + libraries = super."${finalLlvmPackages}".libraries.extend (_: _: { + inherit (pkgs."${finalLlvmPackages}") compiler-rt libcxx libcxxabi; + }); + in + { inherit tools libraries; } // tools // libraries + ); + + inherit binutils binutils-unwrapped; }; - } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) { - # Need to get rid of these when cross-compiling. - inherit binutils binutils-unwrapped; - }; - in import ../generic rec { - name = "stdenv-darwin"; + in + import ../generic rec { + name = "stdenv-darwin"; - inherit config; - inherit (pkgs.stdenv) fetchurlBoot; + inherit config; + inherit (pkgs.stdenv) fetchurlBoot; - buildPlatform = localSystem; - hostPlatform = localSystem; - targetPlatform = localSystem; + buildPlatform = localSystem; + hostPlatform = localSystem; + targetPlatform = localSystem; - preHook = commonPreHook + '' - export NIX_COREFOUNDATION_RPATH=${pkgs.darwin.CF}/Library/Frameworks - export PATH_LOCALE=${pkgs.darwin.locale}/share/locale - ''; + preHook = commonPreHook + '' + export NIX_COREFOUNDATION_RPATH=${pkgs.darwin.CF}/Library/Frameworks + export PATH_LOCALE=${pkgs.darwin.locale}/share/locale + ''; - __stdenvImpureHostDeps = commonImpureHostDeps; - __extraImpureHostDeps = commonImpureHostDeps; + __stdenvImpureHostDeps = commonImpureHostDeps; + __extraImpureHostDeps = commonImpureHostDeps; - initialPath = import ../common-path.nix { inherit pkgs; }; - shell = "${pkgs.bash}/bin/bash"; + initialPath = import ../common-path.nix { inherit pkgs; }; + shell = "${pkgs.bash}/bin/bash"; - cc = pkgs.llvmPackages.libcxxClang.override { - cc = pkgs.llvmPackages.clang-unwrapped; - }; + cc = pkgs."${finalLlvmPackages}".libcxxClang; - extraNativeBuildInputs = []; - extraBuildInputs = [ pkgs.darwin.CF ]; + extraNativeBuildInputs = lib.optionals localSystem.isAarch64 [ + pkgs.updateAutotoolsGnuConfigScriptsHook + ]; - extraAttrs = { - libc = pkgs.darwin.Libsystem; - shellPackage = pkgs.bash; - inherit macosVersionMin appleSdkVersion platform bootstrapTools; - }; + extraBuildInputs = [ pkgs.darwin.CF ]; - allowedRequisites = (with pkgs; [ - xz.out xz.bin libcxx libcxxabi gmp.out gnumake findutils bzip2.out - bzip2.bin llvmPackages.llvm llvmPackages.llvm.lib llvmPackages.compiler-rt llvmPackages.compiler-rt.dev - zlib.out zlib.dev libffi.out coreutils ed diffutils gnutar - gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk - gnugrep llvmPackages.clang-unwrapped llvmPackages.clang-unwrapped.lib patch pcre.out gettext - binutils.bintools darwin.binutils darwin.binutils.bintools - curl.out openssl.out libssh2.out nghttp2.lib libkrb5 - cc.expand-response-params libxml2.out - ]) ++ (with pkgs.darwin; [ - dyld Libsystem CF cctools ICU libiconv locale - ]); - - overrides = lib.composeExtensions persistent (self: super: { - clang = cc; - llvmPackages = super.llvmPackages // { clang = cc; }; - inherit cc; - - darwin = super.darwin // { - inherit (prevStage.darwin) CF; - xnu = super.darwin.xnu.override { inherit (prevStage) python3; }; + extraAttrs = { + libc = pkgs.darwin.Libsystem; + shellPackage = pkgs.bash; + inherit bootstrapTools; }; - }); - }; + + allowedRequisites = (with pkgs; [ + xz.out + xz.bin + gmp.out + gnumake + findutils + bzip2.out + bzip2.bin + zlib.out + zlib.dev + libffi.out + coreutils + ed + diffutils + gnutar + gzip + ncurses.out + ncurses.dev + ncurses.man + gnused + bash + gawk + gnugrep + patch + pcre.out + gettext + binutils.bintools + darwin.binutils + darwin.binutils.bintools + curl.out + openssl.out + libssh2.out + nghttp2.lib + brotli.lib + cc.expand-response-params + libxml2.out + ] ++ lib.optional haveKRB5 libkrb5 + ++ lib.optionals localSystem.isAarch64 [ + pkgs.updateAutotoolsGnuConfigScriptsHook + pkgs.gnu-config + ]) + ++ (with pkgs."${finalLlvmPackages}"; [ + libcxx + libcxx.dev + libcxxabi + libcxxabi.dev + llvm + llvm.lib + compiler-rt + compiler-rt.dev + clang-unwrapped + libclang.dev + libclang.lib + ]) + ++ (with pkgs.darwin; [ + dyld + Libsystem + CF + cctools + ICU + libiconv + locale + libtapi + ] ++ lib.optional useAppleSDKLibs objc4 + ++ lib.optionals doSign [ postLinkSignHook sigtool signingUtils ]); + + overrides = lib.composeExtensions persistent (self: super: { + darwin = super.darwin.overrideScope (_: superDarwin: { + inherit (prevStage.darwin) CF darwin-stubs; + xnu = superDarwin.xnu.override { inherit (prevStage) python3; }; + }); + } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) { + clang = cc; + llvmPackages = super.llvmPackages // { clang = cc; }; + inherit cc; + }); + }; stagesDarwin = [ ({}: stage0) diff --git a/pkgs/stdenv/darwin/fixed-xnu-python3.patch b/pkgs/stdenv/darwin/fixed-xnu-python3.patch new file mode 100644 index 00000000000..9f29376187f --- /dev/null +++ b/pkgs/stdenv/darwin/fixed-xnu-python3.patch @@ -0,0 +1,41 @@ +diff --git a/bsd/kern/makekdebugevents.py b/bsd/kern/makekdebugevents.py +index 73b2db4..d354ba0 100755 +--- a/bsd/kern/makekdebugevents.py ++++ b/bsd/kern/makekdebugevents.py +@@ -5,7 +5,7 @@ + # named kd_events[] or these mappings. + # Required to generate a header file used by DEVELOPMENT and DEBUG kernels. + # +- ++ + import sys + import re + +@@ -21,18 +21,18 @@ code_table = [] + # scan file to generate internal table + with open(trace_code_file, 'rt') as codes: + for line in codes: +- m = id_name_pattern.match(line) +- if m: ++ m = id_name_pattern.match(line) ++ if m: + code_table += [(int(m.group(1),base=16), m.group(2))] + + # emit typedef: +-print "typedef struct {" +-print " uint32_t id;" +-print " const char *name;" +-print "} kd_event_t;" ++print("typedef struct {") ++print(" uint32_t id;") ++print(" const char *name;") ++print("} kd_event_t;") + # emit structure declaration and sorted initialization: +-print "kd_event_t kd_events[] = {" ++print("kd_event_t kd_events[] = {") + for mapping in sorted(code_table, key=lambda x: x[0]): +- print " {0x%x, \"%s\"}," % mapping +-print "};" ++ print(" {0x%x, \"%s\"}," % mapping) ++print("};") + diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix index a299879693c..c62e72f6047 100644 --- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix +++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix @@ -1,9 +1,21 @@ -{ pkgspath ? ../../.., test-pkgspath ? pkgspath, system ? builtins.currentSystem }: - -with import pkgspath { inherit system; }; +{ pkgspath ? ../../.., test-pkgspath ? pkgspath +, system ? builtins.currentSystem, crossSystem ? null, bootstrapFiles ? null +}: + +let cross = if crossSystem != null + then { inherit crossSystem; } + else {}; + custom-bootstrap = if bootstrapFiles != null + then { stdenvStages = args: + let args' = args // { bootstrapFiles = bootstrapFiles; }; + in (import "${pkgspath}/pkgs/stdenv/darwin" args').stagesDarwin; + } + else {}; +in with import pkgspath ({ inherit system; } // cross // custom-bootstrap); let - llvmPackages = llvmPackages_7; + llvmPackages = llvmPackages_11; + storePrefixLen = builtins.stringLength builtins.storeDir; in rec { coreutils_ = coreutils.override (args: { # We want coreutils without ACL support. @@ -18,31 +30,75 @@ in rec { bzip2_ = bzip2.override (args: { linkStatic = true; }); # Avoid messing with libkrb5 and libnghttp2. - curl_ = curl.override (args: { gssSupport = false; http2Support = false; }); + curl_ = curlMinimal.override (args: { gssSupport = false; http2Support = false; }); + + # Avoid stdenv rebuild. + Libsystem_ = (darwin.Libsystem.override (args: + { xnu = darwin.xnu.overrideAttrs (oldAttrs: + { patches = [ ./fixed-xnu-python3.patch ]; }); + })).overrideAttrs (oldAttrs: + { installPhase = oldAttrs.installPhase + '' + cat <<EOF > $out/include/TargetConditionals.h + #ifndef __TARGETCONDITIONALS__ + #define __TARGETCONDITIONALS__ + #define TARGET_OS_MAC 1 + #define TARGET_OS_WIN32 0 + #define TARGET_OS_UNIX 0 + #define TARGET_OS_OSX 1 + #define TARGET_OS_IPHONE 0 + #define TARGET_OS_IOS 0 + #define TARGET_OS_WATCH 0 + #define TARGET_OS_BRIDGE 0 + #define TARGET_OS_TV 0 + #define TARGET_OS_SIMULATOR 0 + #define TARGET_OS_EMBEDDED 0 + #define TARGET_OS_EMBEDDED_OTHER 0 /* Used in configd */ + #define TARGET_IPHONE_SIMULATOR TARGET_OS_SIMULATOR /* deprecated */ + #define TARGET_OS_NANO TARGET_OS_WATCH /* deprecated */ + + #define TARGET_CPU_PPC 0 + #define TARGET_CPU_PPC64 0 + #define TARGET_CPU_68K 0 + #define TARGET_CPU_X86 0 + #define TARGET_CPU_X86_64 1 + #define TARGET_CPU_ARM 0 + #define TARGET_CPU_ARM64 0 + #define TARGET_CPU_MIPS 0 + #define TARGET_CPU_SPARC 0 + #define TARGET_CPU_ALPHA 0 + #define TARGET_RT_MAC_CFM 0 + #define TARGET_RT_MAC_MACHO 1 + #define TARGET_RT_LITTLE_ENDIAN 1 + #define TARGET_RT_BIG_ENDIAN 0 + #define TARGET_RT_64_BIT 1 + #endif /* __TARGETCONDITIONALS__ */ + EOF + ''; + }); build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; - buildInputs = [nukeReferences cpio]; + nativeBuildInputs = [ buildPackages.nukeReferences buildPackages.cpio ] + ++ lib.optionals targetPlatform.isAarch64 [ buildPackages.darwin.sigtool ]; buildCommand = '' - mkdir -p $out/bin $out/lib $out/lib/system + mkdir -p $out/bin $out/lib $out/lib/system $out/lib/darwin - # We're not going to bundle the actual libSystem.dylib; instead we reconstruct it on - # the other side. See the notes in stdenv/darwin/default.nix for more information. - # We also need the .o files for various low-level boot stuff. - cp -d ${darwin.Libsystem}/lib/*.o $out/lib - cp -d ${darwin.Libsystem}/lib/system/*.dylib $out/lib/system + ${lib.optionalString stdenv.targetPlatform.isx86_64 '' + # Copy libSystem's .o files for various low-level boot stuff. + cp -d ${Libsystem_}/lib/*.o $out/lib - # Resolv is actually a link to another package, so let's copy it properly - cp -L ${darwin.Libsystem}/lib/libresolv.9.dylib $out/lib + # Resolv is actually a link to another package, so let's copy it properly + cp -L ${Libsystem_}/lib/libresolv.9.dylib $out/lib - cp -rL ${darwin.Libsystem}/include $out - chmod -R u+w $out/include - cp -rL ${darwin.ICU}/include* $out/include - cp -rL ${libiconv}/include/* $out/include - cp -rL ${gnugrep.pcre.dev}/include/* $out/include - mv $out/include $out/include-Libsystem + cp -rL ${Libsystem_}/include $out + chmod -R u+w $out/include + cp -rL ${darwin.ICU}/include* $out/include + cp -rL ${libiconv}/include/* $out/include + cp -rL ${gnugrep.pcre.dev}/include/* $out/include + mv $out/include $out/include-Libsystem + ''} # Copy coreutils, bash, etc. cp ${coreutils_}/bin/* $out/bin @@ -79,16 +135,32 @@ in rec { # Copy what we need of clang cp -d ${llvmPackages.clang-unwrapped}/bin/clang* $out/bin - - cp -rL ${llvmPackages.clang-unwrapped}/lib/clang $out/lib + cp -rd ${llvmPackages.clang-unwrapped.lib}/lib/* $out/lib cp -d ${llvmPackages.libcxx}/lib/libc++*.dylib $out/lib cp -d ${llvmPackages.libcxxabi}/lib/libc++abi*.dylib $out/lib + cp -d ${llvmPackages.compiler-rt}/lib/darwin/libclang_rt* $out/lib/darwin + cp -d ${llvmPackages.compiler-rt}/lib/libclang_rt* $out/lib cp -d ${llvmPackages.llvm.lib}/lib/libLLVM.dylib $out/lib cp -d ${libffi}/lib/libffi*.dylib $out/lib mkdir $out/include - cp -rd ${llvmPackages.libcxx}/include/c++ $out/include + cp -rd ${llvmPackages.libcxx.dev}/include/c++ $out/include + + ${lib.optionalString targetPlatform.isAarch64 '' + # copy .tbd assembly utils + cp -d ${pkgs.darwin.rewrite-tbd}/bin/rewrite-tbd $out/bin + cp -d ${pkgs.libyaml}/lib/libyaml*.dylib $out/lib + + # copy package extraction tools + cp -d ${pkgs.pbzx}/bin/pbzx $out/bin + cp -d ${pkgs.xar}/lib/libxar*.dylib $out/lib + cp -d ${pkgs.bzip2.out}/lib/libbz2*.dylib $out/lib + + # copy sigtool + cp -d ${pkgs.darwin.sigtool}/bin/sigtool $out/bin + cp -d ${pkgs.darwin.sigtool}/bin/codesign $out/bin + ''} cp -d ${darwin.ICU}/lib/libicu*.dylib $out/lib cp -d ${zlib.out}/lib/libz.* $out/lib @@ -96,20 +168,25 @@ in rec { cp -d ${xz.out}/lib/liblzma*.* $out/lib # Copy binutils. - for i in as ld ar ranlib nm strip otool install_name_tool lipo; do + for i in as ld ar ranlib nm strip otool install_name_tool lipo codesign_allocate; do cp ${cctools_}/bin/$i $out/bin done - cp -rd ${pkgs.darwin.CF}/Library $out + cp -d ${darwin.libtapi}/lib/libtapi* $out/lib + + ${lib.optionalString targetPlatform.isx86_64 '' + cp -rd ${pkgs.darwin.CF}/Library $out + ''} chmod -R u+w $out nuke-refs $out/bin/* rpathify() { - local libs=$(${cctools_}/bin/otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true + local libs=$(${stdenv.cc.targetPrefix}otool -L "$1" | tail -n +2 | grep -o "$NIX_STORE.*-\S*") || true + local newlib for lib in $libs; do - ${cctools_}/bin/install_name_tool -change $lib "@rpath/$(basename $lib)" "$1" + ${stdenv.cc.targetPrefix}install_name_tool -change $lib "@rpath/$(basename "$lib")" "$1" done } @@ -117,20 +194,27 @@ in rec { for i in $out/bin/*; do if test -x $i -a ! -L $i; then chmod +w $i - strip $i || true + ${stdenv.cc.targetPrefix}strip $i || true fi done - for i in $out/bin/* $out/lib/*.dylib $out/lib/clang/*/lib/darwin/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do + for i in $out/bin/* $out/lib/*.dylib $out/lib/darwin/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do if test -x "$i" -a ! -L "$i"; then echo "Adding rpath to $i" rpathify $i fi done + for i in $out/bin/*; do + if test -x "$i" -a ! -L "$i" -a "$(basename $i)" != codesign; then + echo "Adding @executable_path to rpath in $i" + ${stdenv.cc.targetPrefix}install_name_tool -add_rpath '@executable_path/../lib' $i + fi + done + nuke-refs $out/lib/* nuke-refs $out/lib/system/* - nuke-refs $out/lib/clang/*/lib/darwin/* + nuke-refs $out/lib/darwin/* nuke-refs $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation mkdir $out/.pack @@ -144,7 +228,7 @@ in rec { cp ${bzip2_.bin}/bin/bzip2 $out/on-server chmod u+w $out/on-server/* - strip $out/on-server/* + ${stdenv.cc.targetPrefix}strip $out/on-server/* nuke-refs $out/on-server/* (cd $out/pack && (find | cpio -o -H newc)) | bzip2 > $out/on-server/bootstrap-tools.cpio.bz2 @@ -153,7 +237,7 @@ in rec { allowedReferences = []; meta = { - maintainers = [ stdenv.lib.maintainers.copumpkin ]; + maintainers = [ lib.maintainers.copumpkin ]; }; }; @@ -170,6 +254,8 @@ in rec { ''; }; + bootstrapLlvmVersion = llvmPackages.llvm.version; + bootstrapFiles = { sh = "${build}/on-server/sh"; bzip2 = "${build}/on-server/bzip2"; @@ -181,9 +267,6 @@ in rec { unpack = stdenv.mkDerivation (bootstrapFiles // { name = "unpack"; - reexportedLibrariesFile = - ../../os-specific/darwin/apple-source-releases/Libsystem/reexported_libraries; - # This is by necessity a near-duplicate of unpack-bootstrap-tools.sh. If we refer to it directly, # we can't make any changes to it due to our testing stdenv depending on it. Think of this as the # unpack-bootstrap-tools.sh for the next round of bootstrap tools. @@ -206,39 +289,6 @@ in rec { fi done - install_name_tool \ - -id $out/lib/system/libsystem_c.dylib \ - $out/lib/system/libsystem_c.dylib - - install_name_tool \ - -id $out/lib/system/libsystem_kernel.dylib \ - $out/lib/system/libsystem_kernel.dylib - - # TODO: this logic basically duplicates similar logic in the Libsystem expression. Deduplicate them! - libs=$(cat $reexportedLibrariesFile | grep -v '^#') - - for i in $libs; do - if [ "$i" != "/usr/lib/system/libsystem_kernel.dylib" ] && [ "$i" != "/usr/lib/system/libsystem_c.dylib" ]; then - args="$args -reexport_library $i" - fi - done - - ld -macosx_version_min 10.7 \ - -arch x86_64 \ - -dylib \ - -o $out/lib/libSystem.B.dylib \ - -compatibility_version 1.0 \ - -current_version 1226.10.1 \ - -reexport_library $out/lib/system/libsystem_c.dylib \ - -reexport_library $out/lib/system/libsystem_kernel.dylib \ - $args - - ln -s libSystem.B.dylib $out/lib/libSystem.dylib - - for name in c dbm dl info m mx poll proc pthread rpcsvc util gcc_s.10.4 gcc_s.10.5; do - ln -s libSystem.dylib $out/lib/lib$name.dylib - done - ln -s libresolv.9.dylib $out/lib/libresolv.dylib for i in $out/lib/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do @@ -307,7 +357,20 @@ in rec { ${build}/on-server/sh -c 'echo Hello World' - export flags="-idirafter ${unpack}/include-Libsystem --sysroot=${unpack} -L${unpack}/lib" + # This approximates a bootstrap version of libSystem can that be + # assembled via fetchurl. Adapted from main libSystem expression. + mkdir libSystem-boot + cp -vr \ + ${darwin.darwin-stubs}/usr/lib/libSystem.B.tbd \ + ${darwin.darwin-stubs}/usr/lib/system \ + libSystem-boot + + substituteInPlace libSystem-boot/libSystem.B.tbd \ + --replace "/usr/lib/system/" "$PWD/libSystem-boot/system/" + ln -s libSystem.B.tbd libSystem-boot/libSystem.tbd + # End of bootstrap libSystem + + export flags="-idirafter ${unpack}/include-Libsystem --sysroot=${unpack} -L${unpack}/lib -L$PWD/libSystem-boot" export CPP="clang -E $flags" export CC="clang $flags -Wl,-rpath,${unpack}/lib -Wl,-v -Wl,-sdk_version,10.10" @@ -342,9 +405,12 @@ in rec { # The ultimate test: bootstrap a whole stdenv from the tools specified above and get a package set out of it test-pkgs = import test-pkgspath { - inherit system; + # if the bootstrap tools are for another platform, we should be testing + # that platform. + system = if crossSystem != null then crossSystem else system; + stdenvStages = args: let - args' = args // { inherit bootstrapFiles; }; + args' = args // { inherit bootstrapLlvmVersion bootstrapFiles; }; in (import (test-pkgspath + "/pkgs/stdenv/darwin") args').stagesDarwin; }; } diff --git a/pkgs/stdenv/darwin/unpack-bootstrap-tools-aarch64.sh b/pkgs/stdenv/darwin/unpack-bootstrap-tools-aarch64.sh new file mode 100644 index 00000000000..63b72972d71 --- /dev/null +++ b/pkgs/stdenv/darwin/unpack-bootstrap-tools-aarch64.sh @@ -0,0 +1,52 @@ +set -euo pipefail + +# Unpack the bootstrap tools tarball. +echo Unpacking the bootstrap tools... +$mkdir $out +$bzip2 -d < $tarball | (cd $out && $cpio -i) + +export PATH=$out/bin + +# Fix codesign wrapper paths +sed -i \ + -e "1c\ +#!$out/bin/bash" \ + -e "s|[^( ]*\bsigtool\b|$out/bin/sigtool|g" \ + $out/bin/codesign + +updateInstallName() { + local path="$1" + + cp "$path" "$path.new" + install_name_tool -id "$path" "$path.new" + codesign -f -i "$(basename "$path")" -s - "$path.new" + mv -f "$path.new" "$path" +} + +find $out + +ln -s bash $out/bin/sh +ln -s bzip2 $out/bin/bunzip2 + +find $out/lib -type f -name '*.dylib' -print0 | while IFS= read -r -d $'\0' lib; do + updateInstallName "$lib" +done + +# Provide a gunzip script. +cat > $out/bin/gunzip <<EOF +#!$out/bin/sh +exec $out/bin/gzip -d "\$@" +EOF +chmod +x $out/bin/gunzip + +# Provide fgrep/egrep. +echo "#! $out/bin/sh" > $out/bin/egrep +echo "exec $out/bin/grep -E \"\$@\"" >> $out/bin/egrep +echo "#! $out/bin/sh" > $out/bin/fgrep +echo "exec $out/bin/grep -F \"\$@\"" >> $out/bin/fgrep + +cat >$out/bin/dsymutil << EOF +#!$out/bin/sh +EOF + +chmod +x $out/bin/egrep $out/bin/fgrep $out/bin/dsymutil diff --git a/pkgs/stdenv/darwin/unpack-bootstrap-tools.sh b/pkgs/stdenv/darwin/unpack-bootstrap-tools.sh index 0da80ec5ce5..37beeaf28f9 100644 --- a/pkgs/stdenv/darwin/unpack-bootstrap-tools.sh +++ b/pkgs/stdenv/darwin/unpack-bootstrap-tools.sh @@ -17,41 +17,6 @@ for i in $out/bin/*; do fi done -install_name_tool \ - -id $out/lib/system/libsystem_c.dylib \ - $out/lib/system/libsystem_c.dylib - -install_name_tool \ - -id $out/lib/system/libsystem_kernel.dylib \ - $out/lib/system/libsystem_kernel.dylib - -# TODO: this logic basically duplicates similar logic in the Libsystem expression. Deduplicate them! -libs=$(cat $reexportedLibrariesFile | grep -v '^#') - -for i in $libs; do - if [ "$i" != "/usr/lib/system/libsystem_kernel.dylib" ] && [ "$i" != "/usr/lib/system/libsystem_c.dylib" ]; then - args="$args -reexport_library $i" - fi -done - -ld -macosx_version_min 10.7 \ - -arch x86_64 \ - -dylib \ - -o $out/lib/libSystem.B.dylib \ - -compatibility_version 1.0 \ - -current_version 1226.10.1 \ - -reexport_library $out/lib/system/libsystem_c.dylib \ - -reexport_library $out/lib/system/libsystem_kernel.dylib \ - $args - -ln -s libSystem.B.dylib $out/lib/libSystem.dylib - -for name in c dbm dl info m mx poll proc pthread rpcsvc util gcc_s.10.4 gcc_s.10.5; do - ln -s libSystem.dylib $out/lib/lib$name.dylib -done - -ln -s libresolv.9.dylib $out/lib/libresolv.dylib - for i in $out/lib/*.dylib $out/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation; do if test ! -L "$i" -a "$i" != "$out/lib/libSystem*.dylib"; then echo "Patching $i" diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix index b0db1be5f44..d6c59573f2a 100644 --- a/pkgs/stdenv/default.nix +++ b/pkgs/stdenv/default.nix @@ -54,8 +54,10 @@ in aarch64-linux = stagesLinux; mipsel-linux = stagesLinux; powerpc-linux = /* stagesLinux */ stagesNative; + powerpc64-linux = stagesLinux; powerpc64le-linux = stagesLinux; x86_64-darwin = stagesDarwin; + aarch64-darwin = stagesDarwin; x86_64-solaris = stagesNix; i686-cygwin = stagesNative; x86_64-cygwin = stagesNative; diff --git a/pkgs/stdenv/freebsd/default.nix b/pkgs/stdenv/freebsd/default.nix index dbb4a056455..ddcdc6a66e0 100644 --- a/pkgs/stdenv/freebsd/default.nix +++ b/pkgs/stdenv/freebsd/default.nix @@ -1,26 +1,192 @@ { lib -, localSystem, crossSystem, config, overlays +, localSystem, crossSystem, config, overlays, crossOverlays ? [] }: assert crossSystem == localSystem; -let inherit (localSystem) system; in - - +let inherit (localSystem) system; + fetchURL = import <nix/fetchurl.nix>; + trivialBuilder = (import ./trivial-builder.nix); + make = trivialBuilder rec { + inherit (localSystem) system; + name = "make"; + ver = "4.3"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.gz"; + sha256 = "06cfqzpqsvdnsxbysl5p2fgdgxgl9y4p7scpnrfa8z2zgkjdspz0"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + ]; + }; + bash = trivialBuilder rec { + inherit (localSystem) system; + name = "bash"; + ver = "4.4.18"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.gz"; + sha256 = "08vz660768mnnax7n8d4d85jxafwdmsxsi7fh0hzvmafbvn9wkb0"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + ]; + }; + coreutils = trivialBuilder rec { + inherit (localSystem) system; + name = "coreutils"; + ver = "8.31"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "1zg9m79x1i2nifj4kb0waf9x3i5h6ydkypkjnbsb9rnwis8rqypz"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + "--without-gmp" + "--without-libpth-prefix" + ]; + }; + findutils = trivialBuilder rec { + inherit (localSystem) system; + name = "findutils"; + ver = "4.7.0"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "16kqz9yz98dasmj70jwf5py7jk558w96w0vgp3zf9xsqk3gzpzn5"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + "--without-gmp" + "--without-libpth-prefix" + ]; + }; + diffutils = trivialBuilder rec { + inherit (localSystem) system; + name = "diffutils"; + ver = "3.7"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "09isrg0isjinv8c535nxsi1s86wfdfzml80dbw41dj9x3hiad9xk"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + "--without-libsigsegv-prefix" + ]; + }; + grep = trivialBuilder rec { + inherit (localSystem) system; + name = "grep"; + ver = "3.4"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "1yy33kiwrxrwj2nxa4fg15bvmwyghqbs8qwkdvy5phm784f7brjq"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + "--disable-perl-regexp" + "--without-libsegsegv-prefix" + ]; + }; + patch = trivialBuilder rec { + inherit (localSystem) system; + name = "patch"; + ver = "2.7.6"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "1zfqy4rdcy279vwn2z1kbv19dcfw25d2aqy9nzvdkq5bjzd0nqdc"; + }; + gawk = trivialBuilder rec { + inherit (localSystem) system; + name = "gawk"; + ver = "5.0.1"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "15570p7g2x54asvr2fsc56sxzmm08fbk4mzpcs5n92fp9vq8cklf"; + configureArgs = [ "--disable-nls" + "--disable-mpfr" + "--without-libintl-prefix" + "--without-libiconv-prefix" + "--without-libsegsegv-prefix" + ]; + }; + cpio = trivialBuilder rec { + inherit (localSystem) system; + name = "cpio"; + ver = "2.13"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.gz"; + sha256 = "126vyg4a8wcdwh6npgvxy6gq433bzgz3ph37hmjpycc4r7cp0x78"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + ]; + }; + sed = trivialBuilder rec { + inherit (localSystem) system; + name = "sed"; + ver = "4.8"; + url = "https://ftp.gnu.org/gnu/${name}/${name}-${ver}.tar.xz"; + sha256 = "0cznxw73fzv1n3nj2zsq6nf73rvsbxndp444xkpahdqvlzz0r6zp"; + configureArgs = [ "--disable-nls" + "--without-libintl-prefix" + "--without-libiconv-prefix" + ]; + }; + cacert = fetchURL rec { + url = "https://curl.haxx.se/ca/cacert-2020-01-01.pem"; + sha256 = "07q808n307gzaga93abpf6an7c3rd35p18psdc1dd83lspgp1xxd"; + executable = false; + }; + curl = trivialBuilder rec { + inherit (localSystem) system; + name = "curl"; + ver = "7.68.0"; + url = "https://curl.haxx.se/download/${name}-${ver}.tar.xz"; + sha256 = "0nh3j90w6b97wqcgxjfq55qhkz9s38955fbhwzv2fsi7483j895p"; + configureArgs = [ "--disable-nls" + "--disable-ares" + "--disable-debug" + "--disable-ldap" + "--disable-ldaps" + "--disable-rtsp" + "--disable-dict" + "--disable-telnet" + "--disable-tftp" + "--disable-pop3" + "--disable-imap" + "--disable-smb" + "--disable-smtp" + "--disable-gopher" + "--disable-manual" + "--disable-verbose" + "--disable-sspi" + "--disable-tls-srp" + "--disable-unix-sockets" + "--without-brotli" + "--without-gnutls" + "--without-mbedtls" + "--without-wolfssl" + "--without-bearssl" + "--without-libidn2" + "--without-librtmp" + "--without-nghttp2" + "--with-ssl=/usr" + "--with-ca-bundle=${cacert}" + ]; + }; + bashExe = "${bash}/bin/bash"; +in [ ({}: { __raw = true; - bootstrapTools = derivation { + bootstrapTools = derivation ({ inherit system; + inherit make bash coreutils findutils + diffutils grep patch gawk cpio sed + curl; name = "trivial-bootstrap-tools"; - builder = "/usr/local/bin/bash"; + builder = bashExe; args = [ ./trivial-bootstrap.sh ]; - + buildInputs = [ make ]; mkdir = "/bin/mkdir"; ln = "/bin/ln"; - }; + } // lib.optionalAttrs (config.contentAddressedByDefault or false) { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }); }) ({ bootstrapTools, ... }: rec { @@ -52,6 +218,8 @@ let inherit (localSystem) system; in (prevStage: { __raw = true; + inherit (prevStage) bootstrapTools; + stdenv = import ../generic { name = "stdenv-freebsd-boot-0"; inherit config; @@ -66,7 +234,7 @@ let inherit (localSystem) system; in (prevStage: { inherit config overlays; - stdenv = import ../generic { + stdenv = import ../generic rec { name = "stdenv-freebsd-boot-3"; inherit config; @@ -74,20 +242,34 @@ let inherit (localSystem) system; in buildPlatform hostPlatform targetPlatform initialPath shell fetchurlBoot; - cc = import ../../build-support/cc-wrapper { + cc = lib.makeOverridable (import ../../build-support/cc-wrapper) { + inherit lib; nativeTools = true; nativePrefix = "/usr"; nativeLibc = true; stdenvNoCC = prevStage.stdenv; + buildPackages = { + inherit (prevStage) stdenv; + }; cc = { name = "clang-9.9.9"; cc = "/usr"; - outPath = "/usr"; + outPath = prevStage.bootstrapTools; }; isClang = true; + bintools = import ../../build-support/bintools-wrapper { + inherit lib; + stdenvNoCC = prevStage.stdenv; + nativeTools = true; + nativeLibc = true; + propagateDoc = false; + nativePrefix = "/usr"; + bintools = { name = "${name}-binutils"; + outPath = prevStage.bootstrapTools; }; + }; }; - preHook = ''export NIX_NO_SELF_RPATH=1''; + preHook = "export NIX_NO_SELF_RPATH=1"; }; }) diff --git a/pkgs/stdenv/freebsd/trivial-bootstrap.sh b/pkgs/stdenv/freebsd/trivial-bootstrap.sh index fbff4575e5a..34b4dbabc2b 100644 --- a/pkgs/stdenv/freebsd/trivial-bootstrap.sh +++ b/pkgs/stdenv/freebsd/trivial-bootstrap.sh @@ -3,9 +3,9 @@ set -o nounset set -o pipefail echo Building the trivial bootstrap environment... -echo -echo Needed FreeBSD packages: -echo findutils gcpio gawk gnugrep coreutils bash gsed gtar gmake xar binutils gpatch lbzip2 diffutils +#echo +#echo Needed FreeBSD packages: +#echo findutils gcpio gawk gnugrep coreutils bash gsed gtar gmake xar binutils gpatch lbzip2 diffutils $mkdir -p $out/bin @@ -28,14 +28,36 @@ ln () { fi } -ln /usr/local/bin/bash -ln /bin/sh - -ln /usr/local/bin/gmake make +ln $bash/bin/bash +ln $make/bin/make -ln /usr/local/bin/lbzip2 +ln /bin/sh -ln /usr/local/bin/gdiff diff +for i in b2sum base32 base64 basename basenc cat chcon chgrp chmod \ + chown chroot cksum comm cp csplit cut date dd df dir dircolors \ + dirname du echo env expand expr factor false fmt fold install \ + groups head hostid id join kill link ln logname ls md5sum mkdir \ + mkfifo mknod mktemp mv nice nl nohup nproc numfmt od paste pathchk \ + pinky pr printenv printf ptx pwd readlink realpath rm rmdir runcon \ + seq sha1sum sha224sum sha256sum sha384sum sha512sum shred shuf \ + sleep sort split stat stdbuf stty sum sync tac tee test timeout \ + touch tr true truncate tsort tty uname unexpand uniq unlink uptime \ + users vdir wc who whoami yes +do + ln "$coreutils/bin/$i" "$i" +done + +for i in find xargs; do + ln "$findutils/bin/$i" "$i" +done + +for i in diff diff3 sdiff; do + ln "$diffutils/bin/$i" "$i" +done + +for i in grep egrep fgrep; do + ln "$grep/bin/$i" "$i" +done ln /usr/bin/locale @@ -45,160 +67,52 @@ ln /usr/bin/hexdump # for bitcoin ln /usr/bin/bzip2 ln /usr/bin/bunzip2 -ln /usr/bin/bzcat ln /usr/bin/bzip2recover ln /usr/bin/xz ln /usr/bin/unxz -ln /usr/bin/xzcat ln /usr/bin/lzma ln /usr/bin/unlzma -ln /usr/bin/lzcat - -ln /usr/local/bin/gcp cp -ln /usr/local/bin/gdd dd -ln /usr/local/bin/gmv mv -ln /usr/local/bin/grm rm -ln /usr/local/bin/gls ls -ln /bin/ps ps -ln /usr/local/bin/gcat cat -ln /usr/local/bin/gecho echo -ln /usr/local/bin/gexpr expr -ln /usr/local/bin/gtest test -ln /usr/local/bin/gdate date -ln /usr/local/bin/gchmod chmod -ln /usr/local/bin/grmdir rmdir -ln /usr/local/bin/gsleep sleep -ln /bin/hostname hostname - -ln /usr/local/bin/gid id -ln /usr/local/bin/god od -ln /usr/local/bin/gtr tr -ln /usr/local/bin/gwc wc -ln /usr/local/bin/gcut cut -ln /usr/bin/cmp cmp -ln /usr/local/bin/gsed sed -ln /usr/local/bin/gtar tar -ln /usr/local/bin/xar xar -ln /usr/local/bin/gawk awk -ln /usr/local/bin/genv env -ln /usr/local/bin/gtee tee -ln /usr/local/bin/gcomm comm -ln /usr/local/bin/gcpio cpio -ln /usr/local/bin/curl curl -ln /usr/local/bin/gfind find -ln /usr/local/bin/grep grep # other grep is in /usr/bin + +ln /bin/ps +ln /bin/hostname +ln /usr/bin/cmp +ln $sed/bin/sed +ln /usr/bin/tar tar +ln $gawk/bin/gawk +ln $gawk/bin/gawk awk +ln $cpio/bin/cpio +ln $curl/bin/curl curl ln /usr/bin/gzip ln /usr/bin/gunzip -ln /usr/bin/zcat -ln /usr/local/bin/ghead head ln /usr/bin/tail tail # note that we are not using gtail!!! -ln /usr/local/bin/guniq uniq ln /usr/bin/less less -ln /usr/local/bin/gtrue true -# ln /usr/bin/diff diff # we are using gdiff (see above) -ln /usr/local/bin/egrep egrep -ln /usr/local/bin/fgrep fgrep -ln /usr/local/bin/gpatch patch -ln /usr/local/bin/guname uname -ln /usr/local/bin/gtouch touch -ln /usr/local/bin/gsplit split -ln /usr/local/bin/gxargs xargs +ln $patch/bin/patch patch ln /usr/bin/which which -ln /usr/local/bin/ginstall install -ln /usr/local/bin/gbasename basename -ln /usr/local/bin/gdirname dirname -ln /usr/local/bin/greadlink readlink - -ln /usr/local/bin/gln ln -ln /usr/local/bin/gyes yes -ln /usr/local/bin/gwhoami whoami -ln /usr/local/bin/gvdir vdir -ln /usr/local/bin/gusers users -ln /usr/local/bin/guptime uptime -ln /usr/local/bin/gunlink unlink -ln /usr/local/bin/gtty tty -ln /usr/local/bin/gunexpand unexpand -ln /usr/local/bin/gtsort tsort -ln /usr/local/bin/gtruncate truncate -ln /usr/local/bin/gtimeout timeout -ln /usr/local/bin/gtac tac -ln /usr/local/bin/gsync sync -ln /usr/local/bin/gsum sum -ln /usr/local/bin/gstty stty -ln /usr/local/bin/gstdbuf stdbuf -ln /usr/local/bin/gsort sort -ln /usr/local/bin/gruncon runcon -ln /usr/local/bin/gseq seq -ln /usr/local/bin/gsha1sum sha1sum -ln /usr/local/bin/gsha224sum sha224sum -ln /usr/local/bin/gsha256sum sha256sum -ln /usr/local/bin/gsha384sum sha384sum -ln /usr/local/bin/gsha512sum sha512sum -ln /usr/local/bin/gshred shred -ln /usr/local/bin/gshuf shuf -ln /usr/local/bin/grealpath realpath -ln "/usr/local/bin/g[" "[" -ln /usr/local/bin/gbase64 base64 -ln /usr/local/bin/gchcon chcon -ln /usr/local/bin/gchgrp chgrp -ln /usr/local/bin/gchown chown -ln /usr/local/bin/gchroot chroot -ln /usr/local/bin/gcksum cksum -ln /usr/local/bin/gcsplit csplit -ln /usr/local/bin/gdf df -ln /usr/local/bin/gdircolors dircolors -ln /usr/local/bin/gdu du -ln /usr/local/bin/gexpand expand -ln /usr/local/bin/gfactor factor -ln /usr/local/bin/gfalse false -ln /usr/local/bin/gfmt fmt -ln /usr/local/bin/gfold fold -ln /usr/local/bin/ggroups groups -ln /usr/local/bin/ghostid hostid -ln /usr/local/bin/gjoin join -ln /usr/local/bin/gkill kill -ln /usr/local/bin/glink link -ln /usr/local/bin/glogname logname -ln /usr/local/bin/gmd5sum md5sum -ln /usr/local/bin/gmkdir mkdir -ln /usr/local/bin/gmkfifo mkfifo -ln /usr/local/bin/gmknod mknod -ln /usr/local/bin/gmktemp mktemp -ln /usr/local/bin/gnice nice -ln /usr/local/bin/gnl nl -ln /usr/local/bin/gnohup nohup -ln /usr/local/bin/gnproc nproc -ln /usr/local/bin/gnumfmt numfmt -ln /usr/local/bin/gnustat nustat -ln /usr/local/bin/gpaste paste -ln /usr/local/bin/gpathchk pathchk -ln /usr/local/bin/gpinky pinky -ln /usr/local/bin/gpr pr -ln /usr/local/bin/gprintenv printenv -ln /usr/local/bin/gprintf printf -ln /usr/local/bin/gptx ptx -ln /usr/local/bin/gpwd pwd - -# binutils + +## binutils # pkg info -l binutils | grep usr/local/bin -ln /usr/local/bin/addr2line -ln /usr/local/bin/ar -ln /usr/local/bin/as -ln /usr/local/bin/c++filt -ln /usr/local/bin/dwp -ln /usr/local/bin/elfedit -ln /usr/local/bin/gprof -ln /usr/local/bin/ld -ln /usr/local/bin/ld.bfd -ln /usr/local/bin/ld.gold -ln /usr/local/bin/nm -ln /usr/local/bin/objcopy -ln /usr/local/bin/objdump -ln /usr/local/bin/ranlib -ln /usr/local/bin/readelf -ln /usr/local/bin/size -ln /usr/local/bin/strings -ln /usr/local/bin/strip +ln /usr/bin/addr2line +ln /usr/bin/ar +ln /usr/bin/as +ln /usr/bin/c++filt +#ln /usr/bin/dwp +#ln /usr/bin/elfedit +ln /usr/bin/gprof +ln /usr/bin/ld +#ln /usr/bin/ld.bfd +#ln /usr/bin/ld.gold +ln /usr/bin/nm +ln /usr/bin/objcopy +ln /usr/bin/objdump +ln /usr/bin/ranlib +ln /usr/bin/readelf +ln /usr/bin/size +ln /usr/bin/strings +ln /usr/bin/strip + +ln /usr/bin/cc +ln /usr/bin/cpp +ln /usr/bin/c++ #pkg info -l llvm37 | grep usr/local/bin diff --git a/pkgs/stdenv/freebsd/trivial-builder.nix b/pkgs/stdenv/freebsd/trivial-builder.nix new file mode 100644 index 00000000000..64265081f54 --- /dev/null +++ b/pkgs/stdenv/freebsd/trivial-builder.nix @@ -0,0 +1,13 @@ +{ system, name, ver, url, sha256, configureArgs ? [], executable ? false } : + +let fetchURL = import <nix/fetchurl.nix>; + +in derivation { + inherit system configureArgs; + name = "trivial-bootstrap-${name}-${ver}"; + dname = "${name}-${ver}"; + src = fetchURL { + inherit url sha256 executable; + }; + builder = ./trivial-builder.sh; +} diff --git a/pkgs/stdenv/freebsd/trivial-builder.sh b/pkgs/stdenv/freebsd/trivial-builder.sh new file mode 100755 index 00000000000..ac5601b5ba0 --- /dev/null +++ b/pkgs/stdenv/freebsd/trivial-builder.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export PATH=/bin:/sbin:/usr/bin:/usr/sbin + +tar -zxvf $src +cd $dname +mkdir -p $out/bin +./configure --prefix=$out $configureArgs +make +make install diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index c06f17b6fc1..ac62ad30829 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -16,8 +16,8 @@ let allowUnfree = config.allowUnfree or false || builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1"; - whitelist = config.whitelistedLicenses or []; - blacklist = config.blacklistedLicenses or []; + allowlist = config.allowlistedLicenses or config.whitelistedLicenses or []; + blocklist = config.blocklistedLicenses or config.blacklistedLicenses or []; onlyLicenses = list: lib.lists.all (license: @@ -27,19 +27,19 @@ let ) list; areLicenseListsValid = - if lib.mutuallyExclusive whitelist blacklist then - assert onlyLicenses whitelist; assert onlyLicenses blacklist; true + if lib.mutuallyExclusive allowlist blocklist then + assert onlyLicenses allowlist; assert onlyLicenses blocklist; true else - throw "whitelistedLicenses and blacklistedLicenses are not mutually exclusive."; + throw "allowlistedLicenses and blocklistedLicenses are not mutually exclusive."; hasLicense = attrs: attrs ? meta.license; - hasWhitelistedLicense = assert areLicenseListsValid; attrs: - hasLicense attrs && lib.lists.any (l: builtins.elem l whitelist) (lib.lists.toList attrs.meta.license); + hasAllowlistedLicense = assert areLicenseListsValid; attrs: + hasLicense attrs && lib.lists.any (l: builtins.elem l allowlist) (lib.lists.toList attrs.meta.license); - hasBlacklistedLicense = assert areLicenseListsValid; attrs: - hasLicense attrs && lib.lists.any (l: builtins.elem l blacklist) (lib.lists.toList attrs.meta.license); + hasBlocklistedLicense = assert areLicenseListsValid; attrs: + hasLicense attrs && lib.lists.any (l: builtins.elem l blocklist) (lib.lists.toList attrs.meta.license); allowBroken = config.allowBroken or false || builtins.getEnv "NIXPKGS_ALLOW_BROKEN" == "1"; @@ -49,12 +49,24 @@ let isUnfree = licenses: lib.lists.any (l: !l.free or true) licenses; + hasUnfreeLicense = attrs: + hasLicense attrs && + isUnfree (lib.lists.toList attrs.meta.license); + + isMarkedBroken = attrs: attrs.meta.broken or false; + + hasUnsupportedPlatform = attrs: + (!lib.lists.elem hostPlatform.system (attrs.meta.platforms or lib.platforms.all) || + lib.lists.elem hostPlatform.system (attrs.meta.badPlatforms or [])); + + isMarkedInsecure = attrs: (attrs.meta.knownVulnerabilities or []) != []; + # Alow granular checks to allow only some unfree packages # Example: # {pkgs, ...}: # { # allowUnfree = false; - # allowUnfreePredicate = (x: pkgs.lib.hasPrefix "flashplayer-" x.name); + # allowUnfreePredicate = (x: pkgs.lib.hasPrefix "vscode" x.name); # } allowUnfreePredicate = config.allowUnfreePredicate or (x: false); @@ -62,16 +74,15 @@ let # package has an unfree license and is not explicitely allowed by the # `allowUnfreePredicate` function. hasDeniedUnfreeLicense = attrs: + hasUnfreeLicense attrs && !allowUnfree && - hasLicense attrs && - isUnfree (lib.lists.toList attrs.meta.license) && !allowUnfreePredicate attrs; allowInsecureDefaultPredicate = x: builtins.elem (getName x) (config.permittedInsecurePackages or []); allowInsecurePredicate = x: (config.allowInsecurePredicate or allowInsecureDefaultPredicate) x; hasAllowedInsecure = attrs: - (attrs.meta.knownVulnerabilities or []) == [] || + !(isMarkedInsecure attrs) || allowInsecurePredicate attrs || builtins.getEnv "NIXPKGS_ALLOW_INSECURE" == "1"; @@ -80,21 +91,46 @@ let pos_str = meta: meta.position or "«unknown-file»"; remediation = { - unfree = remediate_whitelist "Unfree"; - broken = remediate_whitelist "Broken"; - unsupported = remediate_whitelist "UnsupportedSystem"; - blacklisted = x: ""; + unfree = remediate_allowlist "Unfree" remediate_unfree_predicate; + broken = remediate_allowlist "Broken" (x: ""); + unsupported = remediate_allowlist "UnsupportedSystem" (x: ""); + blocklisted = x: ""; insecure = remediate_insecure; broken-outputs = remediateOutputsToInstall; unknown-meta = x: ""; }; - remediate_whitelist = allow_attr: attrs: + remediation_env_var = allow_attr: { + Unfree = "NIXPKGS_ALLOW_UNFREE"; + Broken = "NIXPKGS_ALLOW_BROKEN"; + UnsupportedSystem = "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM"; + }.${allow_attr}; + remediation_phrase = allow_attr: { + Unfree = "unfree packages"; + Broken = "broken packages"; + UnsupportedSystem = "packages that are unsupported for this system"; + }.${allow_attr}; + remediate_unfree_predicate = attrs: '' - a) For `nixos-rebuild` you can set + + Alternatively you can configure a predicate to allow specific packages: + { nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ + "${lib.getName attrs}" + ]; + } + ''; + + remediate_allowlist = allow_attr: rebuild_amendment: attrs: + '' + a) To temporarily allow ${remediation_phrase allow_attr}, you can use an environment variable + for a single invocation of the nix tools. + + $ export ${remediation_env_var allow_attr}=1 + + b) For `nixos-rebuild` you can set { nixpkgs.config.allow${allow_attr} = true; } in configuration.nix to override this. - - b) For `nix-env`, `nix-build`, `nix-shell` or any other Nix command you can add + ${rebuild_amendment attrs} + c) For `nix-env`, `nix-build`, `nix-shell` or any other Nix command you can add { allow${allow_attr} = true; } to ~/.config/nixpkgs/config.nix. ''; @@ -105,7 +141,7 @@ let Known issues: '' + (lib.concatStrings (map (issue: " - ${issue}\n") attrs.meta.knownVulnerabilities)) + '' - You can install it anyway by whitelisting this package, using the + You can install it anyway by allowing this package, using the following methods: a) To temporarily allow all insecure packages, you can use an environment @@ -167,6 +203,7 @@ let metaTypes = with lib.types; rec { # These keys are documented description = str; + mainProgram = str; longDescription = str; branch = str; homepage = either (listOf str) str; @@ -178,6 +215,9 @@ let platforms = listOf str; hydraPlatforms = listOf str; broken = bool; + unfree = bool; + unsupported = bool; + insecure = bool; # TODO: refactor once something like Profpatsch's types-simple will land # This is currently dead code due to https://github.com/NixOS/nix/issues/2532 tests = attrsOf (mkOptionType { @@ -229,17 +269,22 @@ let # # Return { valid: Bool } and additionally # { reason: String; errormsg: String } if it is not valid, where - # reason is one of "unfree", "blacklisted" or "broken". + # reason is one of "unfree", "blocklisted", "broken", "insecure", ... + # Along with a boolean flag for each reason checkValidity = attrs: - if hasDeniedUnfreeLicense attrs && !(hasWhitelistedLicense attrs) then + { + unfree = hasUnfreeLicense attrs; + broken = isMarkedBroken attrs; + unsupported = hasUnsupportedPlatform attrs; + insecure = isMarkedInsecure attrs; + } + // (if hasDeniedUnfreeLicense attrs && !(hasAllowlistedLicense attrs) then { valid = false; reason = "unfree"; errormsg = "has an unfree license (‘${showLicense attrs.meta.license}’)"; } - else if hasBlacklistedLicense attrs then - { valid = false; reason = "blacklisted"; errormsg = "has a blacklisted license (‘${showLicense attrs.meta.license}’)"; } + else if hasBlocklistedLicense attrs then + { valid = false; reason = "blocklisted"; errormsg = "has a blocklisted license (‘${showLicense attrs.meta.license}’)"; } else if !allowBroken && attrs.meta.broken or false then { valid = false; reason = "broken"; errormsg = "is marked as broken"; } - else if !allowUnsupportedSystem && - (!lib.lists.elem hostPlatform.system (attrs.meta.platforms or lib.platforms.all) || - lib.lists.elem hostPlatform.system (attrs.meta.badPlatforms or [])) then + else if !allowUnsupportedSystem && hasUnsupportedPlatform attrs then { valid = false; reason = "unsupported"; errormsg = "is not supported on ‘${hostPlatform.system}’"; } else if !(hasAllowedInsecure attrs) then { valid = false; reason = "insecure"; errormsg = "is marked as insecure"; } @@ -247,14 +292,14 @@ let { valid = false; reason = "broken-outputs"; errormsg = "has invalid meta.outputsToInstall"; } else let res = checkMeta (attrs.meta or {}); in if res != [] then { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; } - else { valid = true; }; + else { valid = true; }); assertValidity = { meta, attrs }: let validity = checkValidity attrs; in validity // { # Throw an error if trying to evaluate an non-valid derivation handled = if !validity.valid - then handleEvalIssue { inherit meta attrs; } (removeAttrs validity ["valid"]) + then handleEvalIssue { inherit meta attrs; } { inherit (validity) reason errormsg; } else true; }; diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index b5798978690..88ca1b2c790 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -61,12 +61,16 @@ let ] # FIXME this on Darwin; see # https://github.com/NixOS/nixpkgs/commit/94d164dd7#commitcomment-22030369 - ++ lib.optional hostPlatform.isLinux ../../build-support/setup-hooks/audit-tmpdir.sh + ++ lib.optionals hostPlatform.isLinux [ + ../../build-support/setup-hooks/audit-tmpdir.sh + ../../build-support/setup-hooks/move-systemd-user-units.sh + ] ++ [ ../../build-support/setup-hooks/multiple-outputs.sh ../../build-support/setup-hooks/move-sbin.sh ../../build-support/setup-hooks/move-lib64.sh ../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh + ../../build-support/setup-hooks/reproducible-builds.sh # TODO use lib.optional instead (if hasCC then cc else null) ]; @@ -80,6 +84,11 @@ let allowedRequisites = allowedRequisites ++ defaultNativeBuildInputs ++ defaultBuildInputs; } + // lib.optionalAttrs (config.contentAddressedByDefault or false) { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + } // { inherit name; @@ -102,6 +111,8 @@ let '' + lib.optionalString (hostPlatform.isDarwin || (hostPlatform.parsed.kernel.execFormat != lib.systems.parse.execFormats.elf && hostPlatform.parsed.kernel.execFormat != lib.systems.parse.execFormats.macho)) '' export NIX_DONT_SET_RPATH=1 export NIX_NO_SELF_RPATH=1 + '' + lib.optionalString (hostPlatform.isDarwin && hostPlatform.isMacOS) '' + export MACOSX_DEPLOYMENT_TARGET=${hostPlatform.darwinMinVersion} '' # TODO this should be uncommented, but it causes stupid mass rebuilds. I # think the best solution would just be to fixup linux RPATHs so we don't @@ -133,7 +144,7 @@ let # Utility flags to test the type of platform. inherit (hostPlatform) - isDarwin isLinux isSunOS isCygwin isFreeBSD isOpenBSD + isDarwin isLinux isSunOS isCygwin isBSD isFreeBSD isOpenBSD isi686 isx86_32 isx86_64 is32bit is64bit isAarch32 isAarch64 isMips isBigEndian; @@ -148,10 +159,6 @@ let inherit lib config stdenv; }) mkDerivation; - # For convenience, bring in the library functions in lib/ so - # packages don't have to do that themselves. - inherit lib; - inherit fetchurlBoot; inherit overrides; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 5b1c380548f..d6704d59111 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -89,6 +89,10 @@ in rec { , patches ? [] + , __contentAddressed ? + (! attrs ? outputHash) # Fixed-output drvs can't be content addressed too + && (config.contentAddressedByDefault or false) + , ... } @ attrs: let @@ -104,9 +108,16 @@ in rec { ++ depsHostHost ++ depsHostHostPropagated ++ buildInputs ++ propagatedBuildInputs ++ depsTargetTarget ++ depsTargetTargetPropagated) == 0; - dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || (stdenv.noCC or false); + dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || !stdenv.hasCC; supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ]; - defaultHardeningFlags = if stdenv.hostPlatform.isMusl + # Musl-based platforms will keep "pie", other platforms will not. + # If you change this, make sure to update section `{#sec-hardening-in-nixpkgs}` + # in the nixpkgs manual to inform users about the defaults. + defaultHardeningFlags = if stdenv.hostPlatform.isMusl && + # Except when: + # - static aarch64, where compilation works, but produces segfaulting dynamically linked binaries. + # - static armv7l, where compilation fails. + !((stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isAarch32) && stdenv.hostPlatform.isStatic) then supportedHardeningFlags else lib.remove "pie" supportedHardeningFlags; enabledHardeningOptions = @@ -188,15 +199,28 @@ in rec { "__darwinAllowLocalNetworking" "__impureHostDeps" "__propagatedImpureHostDeps" "sandboxProfile" "propagatedSandboxProfile"]) - // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { - name = "${attrs.pname}-${attrs.version}"; - } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { - # Fixed-output derivations like source tarballs shouldn't get a host - # suffix. But we have some weird ones with run-time deps that are - # just used for their side-affects. Those might as well since the - # hash can't be the same. See #32986. - name = "${attrs.name or "${attrs.pname}-${attrs.version}"}-${stdenv.hostPlatform.config}"; - } // { + // (lib.optionalAttrs (attrs ? name || (attrs ? pname && attrs ? version)) { + name = + let + # Indicate the host platform of the derivation if cross compiling. + # Fixed-output derivations like source tarballs shouldn't get a host + # suffix. But we have some weird ones with run-time deps that are + # just used for their side-affects. Those might as well since the + # hash can't be the same. See #32986. + hostSuffix = lib.optionalString + (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix) + "-${stdenv.hostPlatform.config}"; + # Disambiguate statically built packages. This was originally + # introduce as a means to prevent nix-env to get confused between + # nix and nixStatic. This should be also achieved by moving the + # hostSuffix before the version, so we could contemplate removing + # it again. + staticMarker = lib.optionalString stdenv.hostPlatform.isStatic "-static"; + in + if attrs ? name + then attrs.name + hostSuffix + else "${attrs.pname}${staticMarker}${hostSuffix}-${attrs.version}"; + }) // { builder = attrs.realBuilder or stdenv.shell; args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)]; inherit stdenv; @@ -242,6 +266,12 @@ in rec { inherit doCheck doInstallCheck; inherit outputs; + } // lib.optionalAttrs (__contentAddressed) { + inherit __contentAddressed; + # Provide default values for outputHashMode and outputHashAlgo because + # most people won't care about these anyways + outputHashAlgo = attrs.outputHashAlgo or "sha256"; + outputHashMode = attrs.outputHashMode or "recursive"; } // lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) { cmakeFlags = (/**/ if lib.isString cmakeFlags then [cmakeFlags] @@ -251,6 +281,7 @@ in rec { lib.optional (!stdenv.hostPlatform.isRedox) stdenv.hostPlatform.uname.system)}"] ++ lib.optional (stdenv.hostPlatform.uname.processor != null) "-DCMAKE_SYSTEM_PROCESSOR=${stdenv.hostPlatform.uname.processor}" ++ lib.optional (stdenv.hostPlatform.uname.release != null) "-DCMAKE_SYSTEM_VERSION=${stdenv.hostPlatform.release}" + ++ lib.optional (stdenv.hostPlatform.isDarwin) "-DCMAKE_OSX_ARCHITECTURES=${stdenv.hostPlatform.darwinArch}" ++ lib.optional (stdenv.buildPlatform.uname.system != null) "-DCMAKE_HOST_SYSTEM_NAME=${stdenv.buildPlatform.uname.system}" ++ lib.optional (stdenv.buildPlatform.uname.processor != null) "-DCMAKE_HOST_SYSTEM_PROCESSOR=${stdenv.buildPlatform.uname.processor}" ++ lib.optional (stdenv.buildPlatform.uname.release != null) "-DCMAKE_HOST_SYSTEM_VERSION=${stdenv.buildPlatform.uname.release}"; @@ -276,10 +307,10 @@ in rec { in [ "--cross-file=${crossFile}" ] ++ mesonFlags; } // lib.optionalAttrs (attrs.enableParallelBuilding or false) { enableParallelChecking = attrs.enableParallelChecking or true; - } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != []) { + } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != [] || stdenv.hostPlatform.isMusl) { NIX_HARDENING_ENABLE = enabledHardeningOptions; - } // lib.optionalAttrs (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform ? platform.gcc.arch) { - requiredSystemFeatures = attrs.requiredSystemFeatures or [] ++ [ "gccarch-${stdenv.hostPlatform.platform.gcc.arch}" ]; + } // lib.optionalAttrs (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform ? gcc.arch) { + requiredSystemFeatures = attrs.requiredSystemFeatures or [] ++ [ "gccarch-${stdenv.hostPlatform.gcc.arch}" ]; } // lib.optionalAttrs (stdenv.buildPlatform.isDarwin) { inherit __darwinAllowLocalNetworking; # TODO: remove lib.unique once nix has a list canonicalization primitive @@ -309,8 +340,12 @@ in rec { name = attrs.name or "${attrs.pname}-${attrs.version}"; # If the packager hasn't specified `outputsToInstall`, choose a default, - # which is the name of `p.bin or p.out or p`; - # if he has specified it, it will be overridden below in `// meta`. + # which is the name of `p.bin or p.out or p` along with `p.man` when + # present. + # + # If the packager has specified it, it will be overridden below in + # `// meta`. + # # Note: This default probably shouldn't be globally configurable. # Services and users should specify outputs explicitly, # unless they are comfortable with this default. @@ -324,8 +359,9 @@ in rec { # Fill `meta.position` to identify the source location of the package. // lib.optionalAttrs (pos != null) { position = pos.file + ":" + toString pos.line; - # Expose the result of the checks for everyone to see. } // { + # Expose the result of the checks for everyone to see. + inherit (validity) unfree broken unsupported insecure; available = validity.valid && (if config.checkMetaRecursively or false then lib.all (d: d.meta.available or true) references @@ -338,6 +374,32 @@ in rec { validity.handled ({ overrideAttrs = f: mkDerivation (attrs // (f attrs)); + + # A derivation that always builds successfully and whose runtime + # dependencies are the original derivations build time dependencies + # This allows easy building and distributing of all derivations + # needed to enter a nix-shell with + # nix-build shell.nix -A inputDerivation + inputDerivation = derivation (derivationArg // { + # Add a name in case the original drv didn't have one + name = derivationArg.name or "inputDerivation"; + # This always only has one output + outputs = [ "out" ]; + + # Propagate the original builder and arguments, since we override + # them and they might contain references to build inputs + _derivation_original_builder = derivationArg.builder; + _derivation_original_args = derivationArg.args; + + builder = stdenv.shell; + # The bash builtin `export` dumps all current environment variables, + # which is where all build input references end up (e.g. $PATH for + # binaries). By writing this to $out, Nix can find and register + # them as runtime dependencies (since Nix greps for store paths + # through $out to find them) + args = [ "-c" "export > $out" ]; + }); + inherit meta passthru; } // # Pass through extra attributes that are not inputs, but diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh index d19ed342aab..066ef6ed9eb 100644 --- a/pkgs/stdenv/generic/setup.sh +++ b/pkgs/stdenv/generic/setup.sh @@ -1,6 +1,12 @@ set -eu set -o pipefail +if [ -n "${BASH_VERSINFO-}" ] && [ "${BASH_VERSINFO-}" -lt 4 ]; then + echo "Detected Bash version that isn't supported by Nixpkgs (${BASH_VERSION})" + echo "Please install Bash 4 or greater to continue." + exit 1 +fi + if (( "${NIX_DEBUG:-0}" >= 6 )); then set -x fi @@ -157,7 +163,8 @@ addToSearchPathWithCustomDelimiter() { local delimiter="$1" local varName="$2" local dir="$3" - if [ -d "$dir" ]; then + if [[ -d "$dir" && "${!varName:+${delimiter}${!varName}${delimiter}}" \ + != *"${delimiter}${dir}${delimiter}"* ]]; then export "${varName}=${!varName:+${!varName}${delimiter}}${dir}" fi } @@ -483,10 +490,14 @@ activatePackage() { # the transition, we do include everything in thatcase. # # TODO(@Ericson2314): Don't special-case native compilation - if [[ ( -z "${strictDeps-}" || "$hostOffset" -le -1 ) && -d "$pkg/bin" ]]; then + if [[ -z "${strictDeps-}" || "$hostOffset" -le -1 ]]; then addToSearchPath _PATH "$pkg/bin" fi + if [[ "$hostOffset" -le -1 ]]; then + addToSearchPath _XDG_DATA_DIRS "$pkg/share" + fi + if [[ "$hostOffset" -eq 0 && -d "$pkg/bin" ]]; then addToSearchPath _HOST_PATH "$pkg/bin" fi @@ -602,13 +613,16 @@ fi PATH="${_PATH-}${_PATH:+${PATH:+:}}$PATH" HOST_PATH="${_HOST_PATH-}${_HOST_PATH:+${HOST_PATH:+:}}$HOST_PATH" +export XDG_DATA_DIRS="${_XDG_DATA_DIRS-}${_XDG_DATA_DIRS:+${XDG_DATA_DIRS:+:}}${XDG_DATA_DIRS-}" if (( "${NIX_DEBUG:-0}" >= 1 )); then echo "final path: $PATH" echo "final host path: $HOST_PATH" + echo "final data dirs: $XDG_DATA_DIRS" fi unset _PATH unset _HOST_PATH +unset _XDG_DATA_DIRS # Make GNU Make produce nested output. @@ -780,7 +794,7 @@ substituteAllInPlace() { # the environment used for building. dumpVars() { if [ "${noDumpEnvVars:-0}" != 1 ]; then - export >| "$NIX_BUILD_TOP/env-vars" || true + export 2>/dev/null >| "$NIX_BUILD_TOP/env-vars" || true fi } @@ -1037,7 +1051,7 @@ checkPhase() { runHook preCheck if [[ -z "${foundMakefile:-}" ]]; then - echo "no Makefile or custom buildPhase, doing nothing" + echo "no Makefile or custom checkPhase, doing nothing" runHook postCheck return fi @@ -1182,7 +1196,7 @@ installCheckPhase() { runHook preInstallCheck if [[ -z "${foundMakefile:-}" ]]; then - echo "no Makefile or custom buildPhase, doing nothing" + echo "no Makefile or custom installCheckPhase, doing nothing" #TODO(@oxij): should flagsArray influence make -n? elif [[ -z "${installCheckTarget:-}" ]] \ && ! make -n ${makefile:+-f $makefile} ${installCheckTarget:-installcheck} >/dev/null 2>&1; then diff --git a/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix b/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix index 6118585d545..d690f402672 100644 --- a/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix +++ b/pkgs/stdenv/linux/bootstrap-tools-musl/default.nix @@ -1,6 +1,6 @@ -{ system, bootstrapFiles }: +{ system, bootstrapFiles, extraAttrs }: -derivation { +derivation ({ name = "bootstrap-tools"; builder = bootstrapFiles.busybox; @@ -15,4 +15,4 @@ derivation { langC = true; langCC = true; isGNU = true; -} +} // extraAttrs) diff --git a/pkgs/stdenv/linux/bootstrap-tools/default.nix b/pkgs/stdenv/linux/bootstrap-tools/default.nix index 6118585d545..d690f402672 100644 --- a/pkgs/stdenv/linux/bootstrap-tools/default.nix +++ b/pkgs/stdenv/linux/bootstrap-tools/default.nix @@ -1,6 +1,6 @@ -{ system, bootstrapFiles }: +{ system, bootstrapFiles, extraAttrs }: -derivation { +derivation ({ name = "bootstrap-tools"; builder = bootstrapFiles.busybox; @@ -15,4 +15,4 @@ derivation { langC = true; langCC = true; isGNU = true; -} +} // extraAttrs) diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 6c396b27746..1fbd3cba27e 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -42,7 +42,7 @@ assert crossSystem == localSystem; let - inherit (localSystem) system platform; + inherit (localSystem) system; commonPreHook = '' @@ -61,7 +61,16 @@ let # Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...). - bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { inherit system bootstrapFiles; }; + bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { + inherit system bootstrapFiles; + extraAttrs = lib.optionalAttrs + (config.contentAddressedByDefault or false) + { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }; + }; getLibc = stage: stage.${localSystem.libc}; @@ -107,15 +116,11 @@ let bintools = prevStage.binutils; isGNU = true; libc = getLibc prevStage; + inherit lib; inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; }; - extraAttrs = { - # Having the proper 'platform' in all the stdenvs allows getting proper - # linuxHeaders for example. - inherit platform; - }; overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; }; @@ -154,7 +159,8 @@ in # create a dummy Glibc here, which will be used in the stdenv of # stage1. ${localSystem.libc} = self.stdenv.mkDerivation { - name = "bootstrap-stage0-${localSystem.libc}"; + pname = "bootstrap-stage0-${localSystem.libc}"; + version = "bootstrap"; buildCommand = '' mkdir -p $out ln -s ${bootstrapTools}/lib $out/lib @@ -171,6 +177,7 @@ in nativeLibc = false; buildPackages = { }; libc = getLibc self; + inherit lib; inherit (self) stdenvNoCC coreutils gnugrep; bintools = bootstrapTools; }; @@ -251,6 +258,25 @@ in # Rewrap the binutils with the new glibc, so both the next # stage's wrappers use it. libc = getLibc self; + + # Unfortunately, when building gcc in the next stage, its LTO plugin + # would use the final libc but `ld` would use the bootstrap one, + # and that can fail to load. Therefore we upgrade `ld` to use newer libc; + # apparently the interpreter needs to match libc, too. + bintools = self.stdenvNoCC.mkDerivation { + inherit (prevStage.bintools.bintools) name; + dontUnpack = true; + dontBuild = true; + # We wouldn't need to *copy* all, but it's easier and the result is temporary anyway. + installPhase = '' + mkdir -p "$out"/bin + cp -a '${prevStage.bintools.bintools}'/bin/* "$out"/bin/ + chmod +w "$out"/bin/ld.bfd + patchelf --set-interpreter '${getLibc self}'/lib/ld*.so.? \ + --set-rpath "${getLibc self}/lib:$(patchelf --print-rpath "$out"/bin/ld.bfd)" \ + "$out"/bin/ld.bfd + ''; + }; }; }; }) @@ -274,9 +300,13 @@ in gmp = super.gmp.override { stdenv = self.makeStaticLibraries self.stdenv; }; mpfr = super.mpfr.override { stdenv = self.makeStaticLibraries self.stdenv; }; libmpc = super.libmpc.override { stdenv = self.makeStaticLibraries self.stdenv; }; - isl_0_17 = super.isl_0_17.override { stdenv = self.makeStaticLibraries self.stdenv; }; + isl_0_20 = super.isl_0_20.override { stdenv = self.makeStaticLibraries self.stdenv; }; gcc-unwrapped = super.gcc-unwrapped.override { - isl = isl_0_17; + isl = isl_0_20; + # Use a deterministically built compiler + # see https://github.com/NixOS/nixpkgs/issues/108475 for context + reproducibleBuild = true; + profiledCompiler = false; }; }; extraNativeBuildInputs = [ prevStage.patchelf ] ++ @@ -317,6 +347,7 @@ in cc = prevStage.gcc-unwrapped; bintools = self.binutils; libc = getLibc self; + inherit lib; inherit (self) stdenvNoCC coreutils gnugrep; shell = self.bash + "/bin/bash"; }; @@ -369,7 +400,7 @@ in # TODO: remove this! inherit (prevStage) glibc; - inherit platform bootstrapTools; + inherit bootstrapTools; shellPackage = prevStage.bash; }; diff --git a/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix b/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix index d1ee317a2bc..d8ab96952b7 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix @@ -17,6 +17,8 @@ in lib.mapAttrs (n: make) (with lib.systems.examples; { armv6l-musl = muslpi; aarch64-musl = aarch64-multiplatform-musl; riscv64 = riscv64; + powerpc64 = ppc64; + powerpc64-musl = ppc64-musl; powerpc64le = powernv; powerpc64le-musl = musl-power; }) diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index ec5f1092a46..0eee6e692fd 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -47,6 +47,12 @@ in with pkgs; rec { stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; + meta = { + # Increase priority to unblock nixpkgs-unstable + # https://github.com/NixOS/nixpkgs/pull/104679#issuecomment-732267288 + schedulingPriority = 200; + }; + nativeBuildInputs = [ buildPackages.nukeReferences buildPackages.cpio ]; buildCommand = '' @@ -152,7 +158,7 @@ in with pkgs; rec { # These needed for cross but not native tools because the stdenv # GCC has certain things built in statically. See # pkgs/stdenv/linux/default.nix for the details. - cp -d ${isl_0_17.out}/lib/libisl*.so* $out/lib + cp -d ${isl_0_20.out}/lib/libisl*.so* $out/lib '' + '' cp -d ${bzip2.out}/lib/libbz2.so* $out/lib @@ -177,7 +183,7 @@ in with pkgs; rec { nuke-refs $out/lib/* nuke-refs $out/libexec/gcc/*/*/* nuke-refs $out/lib/gcc/*/*/* - nuke-refs $out/lib/gcc/*/*/include-fixed/*/* + nuke-refs $out/lib/gcc/*/*/include-fixed/*{,/*} mkdir $out/.pack mv $out/* $out/.pack @@ -199,6 +205,12 @@ in with pkgs; rec { dist = stdenv.mkDerivation { name = "stdenv-bootstrap-tools"; + meta = { + # Increase priority to unblock nixpkgs-unstable + # https://github.com/NixOS/nixpkgs/pull/104679#issuecomment-732267288 + schedulingPriority = 200; + }; + buildCommand = '' mkdir -p $out/nix-support echo "file tarball ${build}/on-server/bootstrap-tools.tar.xz" >> $out/nix-support/hydra-build-products @@ -212,15 +224,24 @@ in with pkgs; rec { bootstrapTools = runCommand "bootstrap-tools.tar.xz" {} "cp ${build}/on-server/bootstrap-tools.tar.xz $out"; }; - bootstrapTools = if (stdenv.hostPlatform.libc == "glibc") then + bootstrapTools = + let extraAttrs = lib.optionalAttrs + (config.contentAddressedByDefault or false) + { + __contentAddressed = true; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + }; + in + if (stdenv.hostPlatform.libc == "glibc") then import ./bootstrap-tools { inherit (stdenv.buildPlatform) system; # Used to determine where to build - inherit bootstrapFiles; + inherit bootstrapFiles extraAttrs; } else if (stdenv.hostPlatform.libc == "musl") then import ./bootstrap-tools-musl { inherit (stdenv.buildPlatform) system; # Used to determine where to build - inherit bootstrapFiles; + inherit bootstrapFiles extraAttrs; } else throw "unsupported libc"; @@ -246,7 +267,7 @@ in with pkgs; rec { gcc --version '' + lib.optionalString (stdenv.hostPlatform.libc == "glibc") '' - ldlinux=$(echo ${bootstrapTools}/lib/ld-linux*.so.?) + ldlinux=$(echo ${bootstrapTools}/lib/${builtins.baseNameOf binutils.dynamicLinker}) export CPP="cpp -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools}" export CC="gcc -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib" export CXX="g++ -idirafter ${bootstrapTools}/include-glibc -B${bootstrapTools} -Wl,-dynamic-linker,$ldlinux -Wl,-rpath,${bootstrapTools}/lib" diff --git a/pkgs/stdenv/native/default.nix b/pkgs/stdenv/native/default.nix index f6e0df161ad..87862b84bc1 100644 --- a/pkgs/stdenv/native/default.nix +++ b/pkgs/stdenv/native/default.nix @@ -78,7 +78,7 @@ let # A function that builds a "native" stdenv (one that uses tools in # /usr etc.). makeStdenv = - { cc, fetchurl, extraPath ? [], overrides ? (self: super: { }) }: + { cc, fetchurl, extraPath ? [], overrides ? (self: super: { }), extraNativeBuildInputs ? [] }: import ../generic { buildPlatform = localSystem; @@ -94,10 +94,10 @@ let if system == "x86_64-cygwin" then prehookCygwin else prehookBase; - extraNativeBuildInputs = - if system == "i686-cygwin" then extraNativeBuildInputsCygwin else + extraNativeBuildInputs = extraNativeBuildInputs ++ + (if system == "i686-cygwin" then extraNativeBuildInputsCygwin else if system == "x86_64-cygwin" then extraNativeBuildInputsCygwin else - []; + []); initialPath = extraPath ++ path; @@ -129,10 +129,10 @@ in name = "cc-native"; nativeTools = true; nativeLibc = true; - inherit nativePrefix; + inherit lib nativePrefix; bintools = import ../../build-support/bintools-wrapper { name = "bintools"; - inherit stdenvNoCC nativePrefix; + inherit lib stdenvNoCC nativePrefix; nativeTools = true; nativeLibc = true; }; @@ -163,6 +163,7 @@ in inherit (prevStage.stdenv) cc fetchurl; extraPath = [ prevStage.xz ]; overrides = self: super: { inherit (prevStage) xz; }; + extraNativeBuildInputs = if localSystem.isLinux then [ prevStage.patchelf ] else []; }; }) diff --git a/pkgs/stdenv/nix/default.nix b/pkgs/stdenv/nix/default.nix index a8311f49609..2fb19992bc1 100644 --- a/pkgs/stdenv/nix/default.nix +++ b/pkgs/stdenv/nix/default.nix @@ -24,6 +24,7 @@ bootStages ++ [ initialPath = (import ../common-path.nix) { pkgs = prevStage; }; cc = import ../../build-support/cc-wrapper { + inherit lib; nativeTools = false; nativePrefix = lib.optionalString hostPlatform.isSunOS "/usr"; nativeLibc = true; |