diff options
Diffstat (limited to 'pkgs/build-support')
179 files changed, 5537 insertions, 2000 deletions
diff --git a/pkgs/build-support/agda/default.nix b/pkgs/build-support/agda/default.nix index 3c973e8cc0a..44d5ebb9f52 100644 --- a/pkgs/build-support/agda/default.nix +++ b/pkgs/build-support/agda/default.nix @@ -1,13 +1,13 @@ # Builder for Agda packages. -{ stdenv, lib, self, Agda, runCommandNoCC, makeWrapper, writeText, mkShell, ghcWithPackages }: +{ stdenv, lib, self, Agda, runCommandNoCC, makeWrapper, writeText, ghcWithPackages, nixosTests }: with lib.strings; let withPackages' = { pkgs, - ghc ? ghcWithPackages (p: with p; [ ieee ]) + ghc ? ghcWithPackages (p: with p; [ ieee754 ]) }: let pkgs' = if builtins.isList pkgs then pkgs else pkgs self; library-file = writeText "libraries" '' @@ -18,7 +18,10 @@ let in runCommandNoCC "${pname}-${version}" { inherit pname version; nativeBuildInputs = [ makeWrapper ]; - passthru.unwrapped = Agda; + passthru = { + unwrapped = Agda; + tests = { inherit (nixosTests) agda; }; + }; } '' mkdir -p $out/bin makeWrapper ${Agda}/bin/agda $out/bin/agda \ @@ -43,6 +46,7 @@ let defaults = { pname + , meta , buildInputs ? [] , everythingFile ? "./Everything.agda" , libraryName ? pname @@ -70,9 +74,10 @@ let installPhase = if installPhase != null then installPhase else '' runHook preInstall mkdir -p $out - find \( ${concatMapStringsSep " -or " (p: "-name '*.${p}'") (extensions ++ extraExtensions)} \) -exec cp -p --parents -t "$out" {} + + find -not \( -path ${everythingFile} -or -path ${lib.interfaceFile everythingFile} \) -and \( ${concatMapStringsSep " -or " (p: "-name '*.${p}'") (extensions ++ extraExtensions)} \) -exec cp -p --parents -t "$out" {} + runHook postInstall ''; + meta = if meta.broken or false then meta // { hydraPlatforms = lib.platforms.none; } else meta; }; in { diff --git a/pkgs/build-support/agda/lib.nix b/pkgs/build-support/agda/lib.nix new file mode 100644 index 00000000000..976151a8283 --- /dev/null +++ b/pkgs/build-support/agda/lib.nix @@ -0,0 +1,10 @@ +{ lib }: +{ + /* Returns the Agda interface file to a given Agda file. + * + * Examples: + * interfaceFile "Everything.agda" == "Everything.agdai" + * interfaceFile "src/Everything.lagda.tex" == "src/Everything.agdai" + */ + interfaceFile = agdaFile: lib.head (builtins.match ''(.*\.)l?agda(\.(md|org|rst|tex))?'' agdaFile) + "agdai"; +} diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index 5ebbc737e11..cf880677fdd 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -132,7 +132,7 @@ Description: BLAS C implementation Cflags: -I$dev/include Libs: -L$out/lib -lcblas EOF -'' + stdenv.lib.optionalString (blasImplementation == "mkl") '' +'' + lib.optionalString (blasImplementation == "mkl") '' mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ln -s $out/lib/libblas${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index 98b458b778a..7e74eb96b74 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -98,7 +98,7 @@ Description: LAPACK C implementation Cflags: -I$dev/include Libs: -L$out/lib -llapacke EOF -'' + stdenv.lib.optionalString (lapackImplementation == "mkl") '' +'' + lib.optionalString (lapackImplementation == "mkl") '' mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ln -s $out/lib/liblapack${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} diff --git a/pkgs/build-support/appimage/appimage-exec.sh b/pkgs/build-support/appimage/appimage-exec.sh index 82ebdd0bbe4..4ff6802e645 100755 --- a/pkgs/build-support/appimage/appimage-exec.sh +++ b/pkgs/build-support/appimage/appimage-exec.sh @@ -1,4 +1,6 @@ #!@shell@ +# shellcheck shell=bash + if [ -n "$DEBUG" ] ; then set -x fi @@ -13,8 +15,10 @@ unpack() { local out="$2" # https://github.com/AppImage/libappimage/blob/ca8d4b53bed5cbc0f3d0398e30806e0d3adeaaab/src/libappimage/utils/MagicBytesChecker.cpp#L45-L63 - local appimageSignature=$(readelf -h "$src" | awk 'NR==2{print $10$11;}') - local appimageType=$(readelf -h "$src" | awk 'NR==2{print $12;}') + local appimageSignature; + appimageSignature="$(LC_ALL=C readelf -h "$src" | awk 'NR==2{print $10$11;}')" + local appimageType; + appimageType="$(LC_ALL=C readelf -h "$src" | awk 'NR==2{print $12;}')" # check AppImage signature if [ "$appimageSignature" != "4149" ]; then @@ -35,7 +39,7 @@ unpack() { # multiarch offset one-liner using same method as AppImage # see https://gist.github.com/probonopd/a490ba3401b5ef7b881d5e603fa20c93 - offset=$(readelf -h "$src" | awk 'NR==13{e_shoff=$5} NR==18{e_shentsize=$5} NR==19{e_shnum=$5} END{print e_shoff+e_shentsize*e_shnum}') + offset=$(LC_ALL=C readelf -h "$src" | awk 'NR==13{e_shoff=$5} NR==18{e_shentsize=$5} NR==19{e_shnum=$5} END{print e_shoff+e_shentsize*e_shnum}') echo "Uncompress $(basename "$src") of type $appimageType @ offset $offset" unsquashfs -q -d "$out" -o "$offset" "$src" chmod go-w "$out" @@ -71,15 +75,15 @@ apprun() { wrap() { - cd "$APPDIR" || exit # quite same in appimageTools export APPIMAGE_SILENT_INSTALL=1 if [ -n "$APPIMAGE_DEBUG_EXEC" ]; then + cd "$APPDIR" || true exec "$APPIMAGE_DEBUG_EXEC" fi - exec ./AppRun "$@" + exec "$APPDIR/AppRun" "$@" } usage() { diff --git a/pkgs/build-support/appimage/default.nix b/pkgs/build-support/appimage/default.nix index f15ce80d6d9..a759726eb10 100644 --- a/pkgs/build-support/appimage/default.nix +++ b/pkgs/build-support/appimage/default.nix @@ -1,4 +1,5 @@ -{ stdenv +{ lib + , bash , binutils-unwrapped , coreutils @@ -15,7 +16,7 @@ rec { src = ./appimage-exec.sh; isExecutable = true; dir = "bin"; - path = with pkgs; stdenv.lib.makeBinPath [ + path = lib.makeBinPath [ bash binutils-unwrapped coreutils @@ -44,7 +45,7 @@ rec { targetPkgs = pkgs: [ appimage-exec ] ++ defaultFhsEnvArgs.targetPkgs pkgs ++ extraPkgs pkgs; - runScript = "appimage-exec.sh -w ${src}"; + runScript = "appimage-exec.sh -w ${src} --"; } // (removeAttrs args (builtins.attrNames (builtins.functionArgs wrapAppImage)))); wrapType2 = args@{ name, src, extraPkgs ? pkgs: [ ], ... }: wrapAppImage @@ -60,12 +61,12 @@ rec { targetPkgs = pkgs: with pkgs; [ gtk3 bashInteractive - gnome3.zenity + gnome.zenity python2 xorg.xrandr which perl - xdg_utils + xdg-utils iana-etc krb5 ]; @@ -120,7 +121,6 @@ rec { libusb1 udev dbus-glib - libav atk at-spi2-atk libudev0-shim @@ -163,7 +163,6 @@ rec { SDL_mixer SDL2_ttf SDL2_mixer - gstreamer libappindicator-gtk2 libcaca libcanberra @@ -172,7 +171,7 @@ rec { librsvg xorg.libXft libvdpau - alsaLib + alsa-lib harfbuzz e2fsprogs @@ -185,6 +184,7 @@ rec { # libraries not on the upstream include list, but nevertheless expected # by at least one appimage libtool.lib # for Synfigstudio + xorg.libxshmfence # for apple-music-electron at-spi2-core ]; }; diff --git a/pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh b/pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh new file mode 100644 index 00000000000..75d9484846a --- /dev/null +++ b/pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh @@ -0,0 +1,81 @@ +# Unconditionally adding in platform version flags will result in warnings that +# will be treated as errors by some packages. Add any missing flags here. + +# There are two things to be configured: the "platform version" (oldest +# supported version of macos, ios, etc), and the "sdk version". +# +# The modern way of configuring these is to use: +# -platform_version $platform $platform_version $sdk_version" +# +# The old way is still supported, and uses flags like: +# -${platform}_version_min $platform_version +# -sdk_version $sdk_version +# +# If both styles are specified ld will combine them. If multiple versions are +# specified for the same platform, ld will emit an error. +# +# The following adds flags for whichever properties have not already been +# provided. + +havePlatformVersionFlag= +haveDarwinSDKVersion= +haveDarwinPlatformVersion= + +# Roles will set by add-flags.sh, but add-flags.sh can be skipped when the +# cc-wrapper has added the linker flags. Both the cc-wrapper and the binutils +# wrapper mangle the same variable (MACOSX_DEPLOYMENT_TARGET), so if roles are +# empty due to being run through the cc-wrapper then the mangle here is a no-op +# and we still do the right thing. +# +# To be robust, make sure we always have the correct set of roles. +accumulateRoles + +mangleVarSingle @darwinMinVersionVariable@ ${role_suffixes[@]+"${role_suffixes[@]}"} + +n=0 +nParams=${#params[@]} +while (( n < nParams )); do + p=${params[n]} + case "$p" in + # the current platform + -@darwinPlatform@_version_min) + haveDarwinPlatformVersion=1 + ;; + + # legacy aliases + -macosx_version_min|-iphoneos_version_min|-iosmac_version_min|-uikitformac_version_min) + haveDarwinPlatformVersion=1 + ;; + + -sdk_version) + haveDarwinSDKVersion=1 + ;; + + -platform_version) + havePlatformVersionFlag=1 + + # If clang can't determine the sdk version it will pass 0.0.0. This + # has runtime effects so we override this to use the known sdk + # version. + if [ "${params[n+3]-}" = 0.0.0 ]; then + params[n+3]=@darwinSdkVersion@ + fi + ;; + esac + n=$((n + 1)) +done + +# If the caller has set -platform_version, trust they're doing the right thing. +# This will be the typical case for clang in nixpkgs. +if [ ! "$havePlatformVersionFlag" ]; then + if [ ! "$haveDarwinSDKVersion" ] && [ ! "$haveDarwinPlatformVersion" ]; then + # Nothing provided. Use the modern "-platform_version" to set both. + extraBefore+=(-platform_version @darwinPlatform@ "${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@}" @darwinSdkVersion@) + elif [ ! "$haveDarwinSDKVersion" ]; then + # Add missing sdk version + extraBefore+=(-sdk_version @darwinSdkVersion@) + elif [ ! "$haveDarwinPlatformVersion" ]; then + # Add missing platform version + extraBefore+=(-@darwinPlatform@_version_min "${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@}") + fi +fi diff --git a/pkgs/build-support/bintools-wrapper/add-flags.sh b/pkgs/build-support/bintools-wrapper/add-flags.sh index e99beb38158..3b94daba65d 100644 --- a/pkgs/build-support/bintools-wrapper/add-flags.sh +++ b/pkgs/build-support/bintools-wrapper/add-flags.sh @@ -3,6 +3,7 @@ var_templates_list=( NIX_IGNORE_LD_THROUGH_GCC NIX_LDFLAGS NIX_LDFLAGS_BEFORE + NIX_DYNAMIC_LINKER NIX_LDFLAGS_AFTER NIX_LDFLAGS_HARDEN NIX_HARDENING_ENABLE @@ -25,6 +26,10 @@ if [ -e @out@/nix-support/libc-ldflags ]; then NIX_LDFLAGS_@suffixSalt@+=" $(< @out@/nix-support/libc-ldflags)" fi +if [ -z "$NIX_DYNAMIC_LINKER_@suffixSalt@" ] && [ -e @out@/nix-support/ld-set-dynamic-linker ]; then + NIX_DYNAMIC_LINKER_@suffixSalt@="$(< @out@/nix-support/dynamic-linker)" +fi + if [ -e @out@/nix-support/libc-ldflags-before ]; then NIX_LDFLAGS_BEFORE_@suffixSalt@="$(< @out@/nix-support/libc-ldflags-before) $NIX_LDFLAGS_BEFORE_@suffixSalt@" fi diff --git a/pkgs/build-support/bintools-wrapper/darwin-install_name_tool-wrapper.sh b/pkgs/build-support/bintools-wrapper/darwin-install_name_tool-wrapper.sh new file mode 100755 index 00000000000..376a7abfe41 --- /dev/null +++ b/pkgs/build-support/bintools-wrapper/darwin-install_name_tool-wrapper.sh @@ -0,0 +1,49 @@ +#! @shell@ +# shellcheck shell=bash + +set -eu -o pipefail +o posix +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x +fi + +source @signingUtils@ + +extraAfter=() +extraBefore=() +params=("$@") + +input= + +pprev= +prev= +for p in \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} +do + if [ "$pprev" != "-change" ] && [[ "$prev" != -* ]] && [[ "$p" != -* ]]; then + input="$p" + fi + pprev="$prev" + prev="$p" +done + +# Optionally print debug info. +if (( "${NIX_DEBUG:-0}" >= 1 )); then + # Old bash workaround, see above. + echo "extra flags before to @prog@:" >&2 + printf " %q\n" ${extraBefore+"${extraBefore[@]}"} >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" ${params+"${params[@]}"} >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" ${extraAfter+"${extraAfter[@]}"} >&2 +fi + +@prog@ \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} + +sign "$input" diff --git a/pkgs/build-support/bintools-wrapper/darwin-strip-wrapper.sh b/pkgs/build-support/bintools-wrapper/darwin-strip-wrapper.sh new file mode 100755 index 00000000000..a67699547a6 --- /dev/null +++ b/pkgs/build-support/bintools-wrapper/darwin-strip-wrapper.sh @@ -0,0 +1,78 @@ +#! @shell@ +# shellcheck shell=bash + +set -eu -o pipefail +o posix +shopt -s nullglob + +if (( "${NIX_DEBUG:-0}" >= 7 )); then + set -x +fi + +source @signingUtils@ + +extraAfter=() +extraBefore=() +params=("$@") + +output= +inputs=() + +restAreFiles= +prev= +for p in \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} +do + if [ "$restAreFiles" ]; then + inputs+=("$p") + else + case "$prev" in + -s|-R|-d|-arch) + # Unrelated arguments with values + ;; + -o) + # Explicit output + output="$p" + ;; + *) + # Any other orgument either takes no value, or is a file. + if [[ "$p" != -* ]]; then + inputs+=("$p") + fi + ;; + esac + + if [ "$p" == - ]; then + restAreFiles=1 + fi + fi + + prev="$p" +done + +# Optionally print debug info. +if (( "${NIX_DEBUG:-0}" >= 1 )); then + # Old bash workaround, see above. + echo "extra flags before to @prog@:" >&2 + printf " %q\n" ${extraBefore+"${extraBefore[@]}"} >&2 + echo "original flags to @prog@:" >&2 + printf " %q\n" ${params+"${params[@]}"} >&2 + echo "extra flags after to @prog@:" >&2 + printf " %q\n" ${extraAfter+"${extraAfter[@]}"} >&2 +fi + +@prog@ \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} + +if [ "$output" ]; then + # Single explicit output + signIfRequired "$output" +else + # Multiple inputs, rewritten in place + for input in "${inputs[@]}"; do + signIfRequired "$input" + done +fi diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index 786f0f9c598..82d17369743 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -6,16 +6,34 @@ # compiler and the linker just "work". { name ? "" +, lib , stdenvNoCC , bintools ? null, libc ? null, coreutils ? null, shell ? stdenvNoCC.shell, gnugrep ? null +, netbsd ? null, netbsdCross ? null +, sharedLibraryLoader ? + if libc == null then + null + else if stdenvNoCC.targetPlatform.isNetBSD then + if !(targetPackages ? netbsdCross) then + netbsd.ld_elf_so + else if libc != targetPackages.netbsdCross.headers then + targetPackages.netbsdCross.ld_elf_so + else + null + else + lib.getLib libc , nativeTools, noLibc ? false, nativeLibc, nativePrefix ? "" , propagateDoc ? bintools != null && bintools ? man , extraPackages ? [], extraBuildCommands ? "" , buildPackages ? {} +, targetPackages ? {} , useMacosReexportHack ? false + +# Darwin code signing support utilities +, postLinkSignHook ? null, signingUtils ? null }: -with stdenvNoCC.lib; +with lib; assert nativeTools -> !propagateDoc && nativePrefix != ""; assert !nativeTools -> @@ -31,11 +49,11 @@ let # # TODO(@Ericson2314) Make unconditional, or optional but always true by # default. - targetPrefix = stdenv.lib.optionalString (targetPlatform != hostPlatform) + targetPrefix = lib.optionalString (targetPlatform != hostPlatform) (targetPlatform.config + "-"); - bintoolsVersion = stdenv.lib.getVersion bintools; - bintoolsName = stdenv.lib.removePrefix targetPrefix (stdenv.lib.getName bintools); + bintoolsVersion = lib.getVersion bintools; + bintoolsName = lib.removePrefix targetPrefix (lib.getName bintools); libc_bin = if libc == null then null else getBin libc; libc_dev = if libc == null then null else getDev libc; @@ -50,19 +68,22 @@ let # The dynamic linker has different names on different platforms. This is a # shell glob that ought to match it. dynamicLinker = - /**/ if libc == null then null - else if targetPlatform.libc == "musl" then "${libc_lib}/lib/ld-musl-*" - else if targetPlatform.libc == "bionic" then "/system/bin/linker" - else if targetPlatform.libc == "nblibc" then "${libc_lib}/libexec/ld.elf_so" - else if targetPlatform.system == "i686-linux" then "${libc_lib}/lib/ld-linux.so.2" - else if targetPlatform.system == "x86_64-linux" then "${libc_lib}/lib/ld-linux-x86-64.so.2" + /**/ if sharedLibraryLoader == null then null + else if targetPlatform.libc == "musl" then "${sharedLibraryLoader}/lib/ld-musl-*" + else if (targetPlatform.libc == "bionic" && targetPlatform.is32bit) then "/system/bin/linker" + else if (targetPlatform.libc == "bionic" && targetPlatform.is64bit) then "/system/bin/linker64" + else if targetPlatform.libc == "nblibc" then "${sharedLibraryLoader}/libexec/ld.elf_so" + else if targetPlatform.system == "i686-linux" then "${sharedLibraryLoader}/lib/ld-linux.so.2" + else if targetPlatform.system == "x86_64-linux" then "${sharedLibraryLoader}/lib/ld-linux-x86-64.so.2" + else if targetPlatform.system == "powerpc64le-linux" then "${sharedLibraryLoader}/lib/ld64.so.2" # ARM with a wildcard, which can be "" or "-armhf". - else if (with targetPlatform; isAarch32 && isLinux) then "${libc_lib}/lib/ld-linux*.so.3" - else if targetPlatform.system == "aarch64-linux" then "${libc_lib}/lib/ld-linux-aarch64.so.1" - else if targetPlatform.system == "powerpc-linux" then "${libc_lib}/lib/ld.so.1" - else if targetPlatform.isMips then "${libc_lib}/lib/ld.so.1" + else if (with targetPlatform; isAarch32 && isLinux) then "${sharedLibraryLoader}/lib/ld-linux*.so.3" + else if targetPlatform.system == "aarch64-linux" then "${sharedLibraryLoader}/lib/ld-linux-aarch64.so.1" + else if targetPlatform.system == "powerpc-linux" then "${sharedLibraryLoader}/lib/ld.so.1" + else if targetPlatform.isMips then "${sharedLibraryLoader}/lib/ld.so.1" else if targetPlatform.isDarwin then "/usr/lib/dyld" - else if stdenv.lib.hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1" + else if targetPlatform.isFreeBSD then "/libexec/ld-elf.so.1" + else if lib.hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1" else null; expand-response-params = @@ -167,7 +188,7 @@ stdenv.mkDerivation { else if targetPlatform.isWindows then "pe" else "elf" + toString targetPlatform.parsed.cpu.bits; endianPrefix = if targetPlatform.isBigEndian then "big" else "little"; - sep = optionalString (!targetPlatform.isMips && !targetPlatform.isPower) "-"; + sep = optionalString (!targetPlatform.isMips && !targetPlatform.isPower && !targetPlatform.isRiscV) "-"; arch = /**/ if targetPlatform.isAarch64 then endianPrefix + "aarch64" else if targetPlatform.isAarch32 then endianPrefix + "arm" @@ -179,15 +200,20 @@ stdenv.mkDerivation { mips64 = "btsmip"; mips64el = "ltsmip"; }.${targetPlatform.parsed.cpu.name} + else if targetPlatform.isMmix then "mmix" else if targetPlatform.isPower then if targetPlatform.isBigEndian then "ppc" else "lppc" else if targetPlatform.isSparc then "sparc" else if targetPlatform.isMsp430 then "msp430" else if targetPlatform.isAvr then "avr" else if targetPlatform.isAlpha then "alpha" else if targetPlatform.isVc4 then "vc4" + else if targetPlatform.isOr1k then "or1k" + else if targetPlatform.isM68k then "m68k" + else if targetPlatform.isS390 then "s390" + else if targetPlatform.isRiscV then "lriscv" else throw "unknown emulation for platform: ${targetPlatform.config}"; in if targetPlatform.useLLVM or false then "" - else targetPlatform.platform.bfdEmulation or (fmt + sep + arch); + else targetPlatform.bfdEmulation or (fmt + sep + arch); strictDeps = true; depsTargetTargetPropagated = extraPackages; @@ -214,10 +240,10 @@ stdenv.mkDerivation { ## ## Dynamic linker support ## - + '' + + optionalString (sharedLibraryLoader != null) '' if [[ -z ''${dynamicLinker+x} ]]; then echo "Don't know the name of the dynamic linker for platform '${targetPlatform.config}', so guessing instead." >&2 - local dynamicLinker="${libc_lib}/lib/ld*.so.?" + local dynamicLinker="${sharedLibraryLoader}/lib/ld*.so.?" fi '' @@ -234,27 +260,17 @@ stdenv.mkDerivation { if [ -n "''${dynamicLinker-}" ]; then echo $dynamicLinker > $out/nix-support/dynamic-linker - '' + (if targetPlatform.isDarwin then '' - printf "export LD_DYLD_PATH=%q\n" "$dynamicLinker" >> $out/nix-support/setup-hook - '' else '' - if [ -e ${libc_lib}/lib/32/ld-linux.so.2 ]; then - echo ${libc_lib}/lib/32/ld-linux.so.2 > $out/nix-support/dynamic-linker-m32 - fi - '' - # The dynamic linker is passed in `ldflagsBefore' to allow - # explicit overrides of the dynamic linker by callers to ld - # (the *last* value counts, so ours should come first). - + '' - echo -dynamic-linker "$dynamicLinker" >> $out/nix-support/libc-ldflags-before - '') + '' + ${if targetPlatform.isDarwin then '' + printf "export LD_DYLD_PATH=%q\n" "$dynamicLinker" >> $out/nix-support/setup-hook + '' else lib.optionalString (sharedLibraryLoader != null) '' + if [ -e ${sharedLibraryLoader}/lib/32/ld-linux.so.2 ]; then + echo ${sharedLibraryLoader}/lib/32/ld-linux.so.2 > $out/nix-support/dynamic-linker-m32 + fi + touch $out/nix-support/ld-set-dynamic-linker + ''} fi '') - # Ensure consistent LC_VERSION_MIN_MACOSX and remove LC_UUID. - + optionalString stdenv.targetPlatform.isMacOS '' - echo "-macosx_version_min 10.12 -sdk_version 10.12 -no_uuid" >> $out/nix-support/libc-ldflags-before - '' - ## ## User env support ## @@ -304,6 +320,17 @@ stdenv.mkDerivation { done '' + + optionalString stdenv.targetPlatform.isDarwin '' + echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/libc-ldflags + '' + + ### + ### Remove LC_UUID + ### + + optionalString (stdenv.targetPlatform.isDarwin && !(stdenv.cc.bintools.bintools.isGNU or false)) '' + echo "-no_uuid" >> $out/nix-support/libc-ldflags-before + '' + + '' for flags in "$out/nix-support"/*flags*; do substituteInPlace "$flags" --replace $'\n' ' ' @@ -314,6 +341,41 @@ stdenv.mkDerivation { substituteAll ${../wrapper-common/utils.bash} $out/nix-support/utils.bash '' + ### + ### Ensure consistent LC_VERSION_MIN_MACOSX + ### + + optionalString stdenv.targetPlatform.isDarwin ( + let + inherit (stdenv.targetPlatform) + darwinPlatform darwinSdkVersion + darwinMinVersion darwinMinVersionVariable; + in '' + export darwinPlatform=${darwinPlatform} + export darwinMinVersion=${darwinMinVersion} + export darwinSdkVersion=${darwinSdkVersion} + export darwinMinVersionVariable=${darwinMinVersionVariable} + substituteAll ${./add-darwin-ldflags-before.sh} $out/nix-support/add-local-ldflags-before.sh + '' + ) + + ## + ## Code signing on Apple Silicon + ## + + optionalString (targetPlatform.isDarwin && targetPlatform.isAarch64) '' + echo 'source ${postLinkSignHook}' >> $out/nix-support/post-link-hook + + export signingUtils=${signingUtils} + + wrap \ + ${targetPrefix}install_name_tool \ + ${./darwin-install_name_tool-wrapper.sh} \ + "${bintools_bin}/bin/${targetPrefix}install_name_tool" + + wrap \ + ${targetPrefix}strip ${./darwin-strip-wrapper.sh} \ + "${bintools_bin}/bin/${targetPrefix}strip" + '' + ## ## Extra custom steps ## @@ -328,10 +390,10 @@ stdenv.mkDerivation { let bintools_ = if bintools != null then bintools else {}; in (if bintools_ ? meta then removeAttrs bintools.meta ["priority"] else {}) // { description = - stdenv.lib.attrByPath ["meta" "description"] "System binary utilities" bintools_ + lib.attrByPath ["meta" "description"] "System binary utilities" bintools_ + " (wrapper script)"; priority = 10; } // optionalAttrs useMacosReexportHack { - platforms = stdenv.lib.platforms.darwin; + platforms = lib.platforms.darwin; }; } diff --git a/pkgs/build-support/bintools-wrapper/ld-wrapper.sh b/pkgs/build-support/bintools-wrapper/ld-wrapper.sh index 81b5a90edd5..2f96480f80c 100644 --- a/pkgs/build-support/bintools-wrapper/ld-wrapper.sh +++ b/pkgs/build-support/bintools-wrapper/ld-wrapper.sh @@ -23,11 +23,22 @@ fi # Optionally filter out paths not refering to the store. expandResponseParams "$@" + +# NIX_LINK_TYPE is set if ld has been called through our cc wrapper. We take +# advantage of this to avoid both recalculating it, and also repeating other +# processing cc wrapper has already done. +if [[ -n "${NIX_LINK_TYPE_@suffixSalt@:-}" ]]; then + linkType=$NIX_LINK_TYPE_@suffixSalt@ +else + linkType=$(checkLinkType "$@") +fi + if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "${NIX_STORE:-}" - && ( -z "$NIX_IGNORE_LD_THROUGH_GCC_@suffixSalt@" || -z "${NIX_LDFLAGS_SET_@suffixSalt@:-}" ) ]]; then + && ( -z "$NIX_IGNORE_LD_THROUGH_GCC_@suffixSalt@" || -z "${NIX_LINK_TYPE_@suffixSalt@:-}" ) ]]; then rest=() nParams=${#params[@]} declare -i n=0 + while (( "$n" < "$nParams" )); do p=${params[n]} p2=${params[n+1]:-} # handle `p` being last one @@ -55,17 +66,32 @@ if [[ "${NIX_ENFORCE_PURITY:-}" = 1 && -n "${NIX_STORE:-}" params=(${rest+"${rest[@]}"}) fi + source @out@/nix-support/add-hardening.sh extraAfter=() extraBefore=(${hardeningLDFlags[@]+"${hardeningLDFlags[@]}"}) -if [ -z "${NIX_LDFLAGS_SET_@suffixSalt@:-}" ]; then - extraAfter+=($NIX_LDFLAGS_@suffixSalt@) - extraBefore+=($NIX_LDFLAGS_BEFORE_@suffixSalt@) +if [ -z "${NIX_LINK_TYPE_@suffixSalt@:-}" ]; then + extraAfter+=($(filterRpathFlags "$linkType" $NIX_LDFLAGS_@suffixSalt@)) + extraBefore+=($(filterRpathFlags "$linkType" $NIX_LDFLAGS_BEFORE_@suffixSalt@)) + + # By adding dynamic linker to extraBefore we allow the users set their + # own dynamic linker as NIX_LD_FLAGS will override earlier set flags + if [[ "$linkType" == dynamic && -n "$NIX_DYNAMIC_LINKER_@suffixSalt@" ]]; then + extraBefore+=("-dynamic-linker" "$NIX_DYNAMIC_LINKER_@suffixSalt@") + fi +fi + +extraAfter+=($(filterRpathFlags "$linkType" $NIX_LDFLAGS_AFTER_@suffixSalt@)) + +# These flags *must not* be pulled up to -Wl, flags, so they can't go in +# add-flags.sh. They must always be set, so must not be disabled by +# NIX_LDFLAGS_SET. +if [ -e @out@/nix-support/add-local-ldflags-before.sh ]; then + source @out@/nix-support/add-local-ldflags-before.sh fi -extraAfter+=($NIX_LDFLAGS_AFTER_@suffixSalt@) # Specify the target emulation if nothing is passed in ("-m" overrides this # environment variable). Ensures we never blindly fallback on targeting the host @@ -83,6 +109,8 @@ declare -a libDirs declare -A libs declare -i relocatable=0 link32=0 +linkerOutput="a.out" + if [ "$NIX_DONT_SET_RPATH_@suffixSalt@" != 1 ] \ || [ "$NIX_SET_BUILD_ID_@suffixSalt@" = 1 ] \ @@ -134,7 +162,25 @@ then done fi -if [ -e "@out@/nix-support/dynamic-linker-m32" ] && (( "$link32" )); then +# Determine linkerOutput +prev= +for p in \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} +do + case "$prev" in + -o) + # Informational for post-link-hook + linkerOutput="$p" + ;; + *) + ;; + esac + prev="$p" +done + +if [[ "$link32" == "1" && "$linkType" == dynamic && -e "@out@/nix-support/dynamic-linker-m32" ]]; then # We have an alternate 32-bit linker and we're producing a 32-bit ELF, let's # use it. extraAfter+=( @@ -144,7 +190,7 @@ if [ -e "@out@/nix-support/dynamic-linker-m32" ] && (( "$link32" )); then fi # Add all used dynamic libraries to the rpath. -if [ "$NIX_DONT_SET_RPATH_@suffixSalt@" != 1 ]; then +if [[ "$NIX_DONT_SET_RPATH_@suffixSalt@" != 1 && "$linkType" != static-pie ]]; then # For each directory in the library search path (-L...), # see if it contains a dynamic library used by a -l... flag. If # so, add the directory to the rpath. @@ -204,7 +250,11 @@ fi PATH="$path_backup" # Old bash workaround, see above. -exec @prog@ \ +@prog@ \ ${extraBefore+"${extraBefore[@]}"} \ ${params+"${params[@]}"} \ ${extraAfter+"${extraAfter[@]}"} + +if [ -e "@out@/nix-support/post-link-hook" ]; then + source @out@/nix-support/post-link-hook +fi diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index bbcbc4e2e11..198b9c3f617 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -9,7 +9,7 @@ let in args@{ - name + name ? "${args.pname}-${args.version}" , bazel ? bazelPkg , bazelFlags ? [] , bazelBuildFlags ? [] @@ -35,8 +35,14 @@ args@{ # required for the build as configured, rather than fetching all the dependencies # which may not work in some situations (e.g. Java code which ends up relying on # Debian-specific /usr/share/java paths, but doesn't in the configured build). -, fetchConfigured ? false - +, fetchConfigured ? true + +# Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE / +# NIX_LDFLAGS. This is necessary when using a custom toolchain which +# Bazel wants all headers / libraries to come from, like when using +# CROSSTOOL. Weirdly, we can still get the flags through the wrapped +# compiler. +, dontAddBazelOpts ? false , ... }: @@ -102,8 +108,8 @@ in stdenv.mkDerivation (fBuildAttrs // { rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker} ${if removeRulesCC then "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}" else ""} rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker} - ${if removeLocalConfigCc then "rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker}" else ""} - ${if removeLocal then "rm -rf $bazelOut/external/{local_*,\@local_*.marker}" else ""} + ${if removeLocalConfigCc then "rm -rf $bazelOut/external/{local_config_cc,\\@local_config_cc.marker}" else ""} + ${if removeLocal then "rm -rf $bazelOut/external/{local_*,\\@local_*.marker}" else ""} # Clear markers find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \; @@ -120,7 +126,7 @@ in stdenv.mkDerivation (fBuildAttrs // { find $bazelOut/external -maxdepth 1 -type l | while read symlink; do name="$(basename "$symlink")" rm "$symlink" - test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" + test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" || true done # Patching symlinks to remove build directory reference @@ -166,10 +172,14 @@ in stdenv.mkDerivation (fBuildAttrs // { chmod -R +w $bazelOut find $bazelOut -type l | while read symlink; do - ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink" + if [[ $(readlink "$symlink") == *NIX_BUILD_TOP* ]]; then + ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink" + fi done '' + fBuildAttrs.preConfigure or ""; + inherit dontAddBazelOpts; + buildPhase = fBuildAttrs.buildPhase or '' runHook preBuild @@ -181,20 +191,22 @@ in stdenv.mkDerivation (fBuildAttrs // { # copts=() host_copts=() - for flag in $NIX_CFLAGS_COMPILE; do - copts+=( "--copt=$flag" ) - host_copts+=( "--host_copt=$flag" ) - done - for flag in $NIX_CXXSTDLIB_COMPILE; do - copts+=( "--copt=$flag" ) - host_copts+=( "--host_copt=$flag" ) - done linkopts=() host_linkopts=() - for flag in $NIX_LDFLAGS; do - linkopts+=( "--linkopt=-Wl,$flag" ) - host_linkopts+=( "--host_linkopt=-Wl,$flag" ) - done + if [ -z "''${dontAddBazelOpts:-}" ]; then + for flag in $NIX_CFLAGS_COMPILE; do + copts+=( "--copt=$flag" ) + host_copts+=( "--host_copt=$flag" ) + done + for flag in $NIX_CXXSTDLIB_COMPILE; do + copts+=( "--copt=$flag" ) + host_copts+=( "--host_copt=$flag" ) + done + for flag in $NIX_LDFLAGS; do + linkopts+=( "--linkopt=-Wl,$flag" ) + host_linkopts+=( "--host_linkopt=-Wl,$flag" ) + done + fi BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \ USER=homeless-shelter \ diff --git a/pkgs/build-support/build-dotnet-package/default.nix b/pkgs/build-support/build-dotnet-package/default.nix index dae9ed888c7..440b10044f0 100644 --- a/pkgs/build-support/build-dotnet-package/default.nix +++ b/pkgs/build-support/build-dotnet-package/default.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, makeWrapper, pkgconfig, mono, dotnetbuildhelpers }: +{ stdenv, lib, makeWrapper, pkg-config, mono, dotnetbuildhelpers }: attrsOrig @ { baseName @@ -19,7 +19,7 @@ attrsOrig @ attrs = { name = "${baseName}-${version}"; - nativeBuildInputs = [ pkgconfig ]; + nativeBuildInputs = [ pkg-config ]; buildInputs = [ mono dotnetbuildhelpers diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix index c7cfd27d3fa..23718bf636c 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix @@ -1,28 +1,43 @@ -{ callPackage, runCommandLocal, writeShellScriptBin, stdenv, coreutils, bubblewrap }: +{ lib, callPackage, runCommandLocal, writeShellScriptBin, glibc, pkgsi686Linux, coreutils, bubblewrap }: let buildFHSEnv = callPackage ./env.nix { }; in args @ { - name, - runScript ? "bash", - extraInstallCommands ? "", - meta ? {}, - passthru ? {}, - ... + name +, runScript ? "bash" +, extraInstallCommands ? "" +, meta ? {} +, passthru ? {} +, unshareUser ? true +, unshareIpc ? true +, unsharePid ? true +, unshareNet ? false +, unshareUts ? true +, unshareCgroup ? true +, dieWithParent ? true +, ... }: with builtins; let + buildFHSEnv = callPackage ./env.nix { }; + env = buildFHSEnv (removeAttrs args [ - "runScript" "extraInstallCommands" "meta" "passthru" + "runScript" "extraInstallCommands" "meta" "passthru" "dieWithParent" + "unshareUser" "unshareCgroup" "unshareUts" "unshareNet" "unsharePid" "unshareIpc" ]); - chrootenv = callPackage ./chrootenv {}; - etcBindFlags = let files = [ # NixOS Compatibility "static" + "nix" # mainly for nixUnstable users, but also for access to nix/netrc + # Shells + "bashrc" + "zshenv" + "zshrc" + "zinputrc" + "zprofile" # Users, Groups, NSS "passwd" "group" @@ -30,6 +45,8 @@ let "hosts" "resolv.conf" "nsswitch.conf" + # User profiles + "profiles" # Sudo & Su "login.defs" "sudoers" @@ -45,60 +62,115 @@ let # Fonts "fonts" # ALSA + "alsa" "asound.conf" # SSL "ssl/certs" "pki" ]; - in concatStringsSep " \\\n " + in concatStringsSep "\n " (map (file: "--ro-bind-try /etc/${file} /etc/${file}") files); + # Create this on the fly instead of linking from /nix + # The container might have to modify it and re-run ldconfig if there are + # issues running some binary with LD_LIBRARY_PATH + createLdConfCache = '' + cat > /etc/ld.so.conf <<EOF + /lib + /lib/x86_64-linux-gnu + /lib64 + /usr/lib + /usr/lib/x86_64-linux-gnu + /usr/lib64 + /lib/i386-linux-gnu + /lib32 + /usr/lib/i386-linux-gnu + /usr/lib32 + EOF + ldconfig &> /dev/null + ''; init = run: writeShellScriptBin "${name}-init" '' source /etc/profile + ${createLdConfCache} exec ${run} "$@" ''; bwrapCmd = { initArgs ? "" }: '' - blacklist="/nix /dev /proc /etc" - ro_mounts="" + blacklist=(/nix /dev /proc /etc) + ro_mounts=() + symlinks=() for i in ${env}/*; do path="/''${i##*/}" if [[ $path == '/etc' ]]; then - continue + : + elif [[ -L $i ]]; then + symlinks+=(--symlink "$(${coreutils}/bin/readlink "$i")" "$path") + blacklist+=("$path") + else + ro_mounts+=(--ro-bind "$i" "$path") + blacklist+=("$path") fi - ro_mounts="$ro_mounts --ro-bind $i $path" - blacklist="$blacklist $path" done if [[ -d ${env}/etc ]]; then for i in ${env}/etc/*; do path="/''${i##*/}" - ro_mounts="$ro_mounts --ro-bind $i /etc$path" + # NOTE: we're binding /etc/fonts and /etc/ssl/certs from the host so we + # don't want to override it with a path from the FHS environment. + if [[ $path == '/fonts' || $path == '/ssl' ]]; then + continue + fi + ro_mounts+=(--ro-bind "$i" "/etc$path") done fi - auto_mounts="" + declare -a auto_mounts # loop through all directories in the root for dir in /*; do # if it is a directory and it is not in the blacklist - if [[ -d "$dir" ]] && grep -v "$dir" <<< "$blacklist" >/dev/null; then + if [[ -d "$dir" ]] && [[ ! "''${blacklist[@]}" =~ "$dir" ]]; then # add it to the mount list - auto_mounts="$auto_mounts --bind $dir $dir" + auto_mounts+=(--bind "$dir" "$dir") fi done - exec ${bubblewrap}/bin/bwrap \ - --dev-bind /dev /dev \ - --proc /proc \ - --chdir "$(pwd)" \ - --unshare-all \ - --share-net \ - --die-with-parent \ - --ro-bind /nix /nix \ - ${etcBindFlags} \ - $ro_mounts \ - $auto_mounts \ + cmd=( + ${bubblewrap}/bin/bwrap + --dev-bind /dev /dev + --proc /proc + --chdir "$(pwd)" + ${lib.optionalString unshareUser "--unshare-user"} + ${lib.optionalString unshareIpc "--unshare-ipc"} + ${lib.optionalString unsharePid "--unshare-pid"} + ${lib.optionalString unshareNet "--unshare-net"} + ${lib.optionalString unshareUts "--unshare-uts"} + ${lib.optionalString unshareCgroup "--unshare-cgroup"} + ${lib.optionalString dieWithParent "--die-with-parent"} + --ro-bind /nix /nix + # Our glibc will look for the cache in its own path in `/nix/store`. + # As such, we need a cache to exist there, because pressure-vessel + # depends on the existence of an ld cache. However, adding one + # globally proved to be a bad idea (see #100655), the solution we + # settled on being mounting one via bwrap. + # Also, the cache needs to go to both 32 and 64 bit glibcs, for games + # of both architectures to work. + --tmpfs ${glibc}/etc \ + --symlink /etc/ld.so.conf ${glibc}/etc/ld.so.conf \ + --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \ + --ro-bind ${glibc}/etc/rpc ${glibc}/etc/rpc \ + --remount-ro ${glibc}/etc \ + --tmpfs ${pkgsi686Linux.glibc}/etc \ + --symlink /etc/ld.so.conf ${pkgsi686Linux.glibc}/etc/ld.so.conf \ + --symlink /etc/ld.so.cache ${pkgsi686Linux.glibc}/etc/ld.so.cache \ + --ro-bind ${pkgsi686Linux.glibc}/etc/rpc ${pkgsi686Linux.glibc}/etc/rpc \ + --remount-ro ${pkgsi686Linux.glibc}/etc \ + ${etcBindFlags} + "''${ro_mounts[@]}" + "''${symlinks[@]}" + "''${auto_mounts[@]}" ${init runScript}/bin/${name}-init ${initArgs} + ) + exec "''${cmd[@]}" ''; bin = writeShellScriptBin name (bwrapCmd { initArgs = ''"$@"''; }); diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix index 8b2d46c4ae9..b9c719a4c78 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix @@ -1,4 +1,4 @@ -{ stdenv, buildEnv, writeText, pkgs, pkgsi686Linux }: +{ stdenv, lib, buildEnv, writeText, writeShellScriptBin, pkgs, pkgsi686Linux }: { name, profile ? "" , targetPkgs ? pkgs: [], multiPkgs ? pkgs: [] @@ -49,6 +49,9 @@ let [ (toString gcc.cc.lib) ]; + ldconfig = writeShellScriptBin "ldconfig" '' + exec ${pkgs.glibc.bin}/bin/ldconfig -f /etc/ld.so.conf -C /etc/ld.so.cache "$@" + ''; etcProfile = writeText "profile" '' export PS1='${name}-chrootenv:\u@\h:\w\$ ' export LOCALE_ARCHIVE='/usr/lib/locale/locale-archive' @@ -86,7 +89,8 @@ let # Composes a /usr-like directory structure staticUsrProfileTarget = buildEnv { name = "${name}-usr-target"; - paths = [ etcPkg ] ++ basePkgs ++ targetPaths; + # ldconfig wrapper must come first so it overrides the original ldconfig + paths = [ etcPkg ldconfig ] ++ basePkgs ++ targetPaths; extraOutputsToInstall = [ "out" "lib" "bin" ] ++ extraOutputsToInstall; ignoreCollisions = true; }; @@ -132,7 +136,20 @@ let mkdir -m0755 usr cd usr ${setupLibDirs} - for i in bin sbin share include; do + ${lib.optionalString isMultiBuild '' + if [ -d "${staticUsrProfileMulti}/share" ]; then + cp -rLf ${staticUsrProfileMulti}/share share + fi + ''} + if [ -d "${staticUsrProfileTarget}/share" ]; then + if [ -d share ]; then + chmod -R 755 share + cp -rLTf ${staticUsrProfileTarget}/share share + else + cp -rLf ${staticUsrProfileTarget}/share share + fi + fi + for i in bin sbin include; do if [ -d "${staticUsrProfileTarget}/$i" ]; then cp -rsHf "${staticUsrProfileTarget}/$i" "$i" fi diff --git a/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c b/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c index dcb2e97aa93..27e70e3fe5c 100644 --- a/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c +++ b/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c @@ -56,6 +56,7 @@ void bind_mount_item(const gchar *host, const gchar *guest, const gchar *name) { void bind(const gchar *host, const gchar *guest) { mount_tmpfs(guest); + pivot_host(guest); g_autofree gchar *host_dir = g_build_filename("/host", host, NULL); @@ -105,7 +106,11 @@ int main(gint argc, gchar **argv) { uid_t uid = getuid(); gid_t gid = getgid(); - if (unshare(CLONE_NEWNS | CLONE_NEWUSER) < 0) { + int namespaces = CLONE_NEWNS; + if (uid != 0) { + namespaces |= CLONE_NEWUSER; + } + if (unshare(namespaces) < 0) { int unshare_errno = errno; g_message("Requires Linux version >= 3.19 built with CONFIG_USER_NS"); @@ -116,9 +121,14 @@ int main(gint argc, gchar **argv) { fail("unshare", unshare_errno); } - spit("/proc/self/setgroups", "deny"); - spit("/proc/self/uid_map", "%d %d 1", uid, uid); - spit("/proc/self/gid_map", "%d %d 1", gid, gid); + // hide all mounts we do from the parent + fail_if(mount(0, "/", 0, MS_PRIVATE | MS_REC, 0)); + + if (uid != 0) { + spit("/proc/self/setgroups", "deny"); + spit("/proc/self/uid_map", "%d %d 1", uid, uid); + spit("/proc/self/gid_map", "%d %d 1", gid, gid); + } // If there is a /host directory, assume this is nested chrootenv and use it as host instead. gboolean nested_host = g_file_test("/host", G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR); diff --git a/pkgs/build-support/build-fhs-userenv/chrootenv/default.nix b/pkgs/build-support/build-fhs-userenv/chrootenv/default.nix index 70a7a43bd39..15d7b315358 100644 --- a/pkgs/build-support/build-fhs-userenv/chrootenv/default.nix +++ b/pkgs/build-support/build-fhs-userenv/chrootenv/default.nix @@ -1,13 +1,13 @@ -{ stdenv, meson, ninja, pkgconfig, glib }: +{ lib, stdenv, meson, ninja, pkg-config, glib }: stdenv.mkDerivation { name = "chrootenv"; src = ./.; - nativeBuildInputs = [ meson ninja pkgconfig ]; + nativeBuildInputs = [ meson ninja pkg-config ]; buildInputs = [ glib ]; - meta = with stdenv.lib; { + meta = with lib; { description = "Setup mount/user namespace for FHS emulation"; license = licenses.mit; maintainers = with maintainers; [ yegortimoshenko ]; diff --git a/pkgs/build-support/build-fhs-userenv/env.nix b/pkgs/build-support/build-fhs-userenv/env.nix index 083e7617b50..226904f311b 100644 --- a/pkgs/build-support/build-fhs-userenv/env.nix +++ b/pkgs/build-support/build-fhs-userenv/env.nix @@ -81,6 +81,9 @@ let # compatibility with NixOS ln -s /host/etc/static static + # symlink nix config + ln -s /host/etc/nix nix + # symlink some NSS stuff ln -s /host/etc/passwd passwd ln -s /host/etc/group group @@ -89,6 +92,9 @@ let ln -s /host/etc/resolv.conf resolv.conf ln -s /host/etc/nsswitch.conf nsswitch.conf + # symlink user profiles + ln -s /host/etc/profiles profiles + # symlink sudo and su stuff ln -s /host/etc/login.defs login.defs ln -s /host/etc/sudoers sudoers diff --git a/pkgs/build-support/build-pecl.nix b/pkgs/build-support/build-pecl.nix index d75d3cf943a..8168c7a3de1 100644 --- a/pkgs/build-support/build-pecl.nix +++ b/pkgs/build-support/build-pecl.nix @@ -2,12 +2,12 @@ { pname , version -, internalDeps ? [] -, peclDeps ? [] -, buildInputs ? [] -, nativeBuildInputs ? [] +, internalDeps ? [ ] +, peclDeps ? [ ] +, buildInputs ? [ ] +, nativeBuildInputs ? [ ] , postPhpize ? "" -, makeFlags ? [] +, makeFlags ? [ ] , src ? fetchurl { url = "http://pecl.php.net/get/${pname}-${version}.tgz"; inherit (args) sha256; @@ -33,4 +33,5 @@ stdenv.mkDerivation (args // { (dep: "mkdir -p ext; ln -s ${dep.dev}/include ext/${dep.extensionName}") internalDeps} ''; + checkPhase = "NO_INTERACTON=yes make test"; }) diff --git a/pkgs/build-support/buildenv/default.nix b/pkgs/build-support/buildenv/default.nix index 7f2427777f9..c2186cf6bfa 100644 --- a/pkgs/build-support/buildenv/default.nix +++ b/pkgs/build-support/buildenv/default.nix @@ -36,8 +36,9 @@ lib.makeOverridable , # Shell commands to run after building the symlink tree. postBuild ? "" -, # Additional inputs. Handy e.g. if using makeWrapper in `postBuild`. - buildInputs ? [] +# Additional inputs +, nativeBuildInputs ? [] # Handy e.g. if using makeWrapper in `postBuild`. +, buildInputs ? [] , passthru ? {} , meta ? {} @@ -53,7 +54,8 @@ in runCommand name rec { inherit manifest ignoreCollisions checkCollisionContents passthru - meta pathsToLink extraPrefix postBuild buildInputs; + meta pathsToLink extraPrefix postBuild + nativeBuildInputs buildInputs; pkgs = builtins.toJSON (map (drv: { paths = # First add the usual output(s): respect if user has chosen explicitly, @@ -72,7 +74,7 @@ runCommand name preferLocalBuild = true; allowSubstitutes = false; # XXX: The size is somewhat arbitrary - passAsFile = if builtins.stringLength pkgs >= 128*1024 then [ "pkgs" ] else null; + passAsFile = if builtins.stringLength pkgs >= 128*1024 then [ "pkgs" ] else [ ]; } '' ${buildPackages.perl}/bin/perl -w ${builder} diff --git a/pkgs/build-support/cc-wrapper/add-flags.sh b/pkgs/build-support/cc-wrapper/add-flags.sh index 94589131b70..a6e991914e6 100644 --- a/pkgs/build-support/cc-wrapper/add-flags.sh +++ b/pkgs/build-support/cc-wrapper/add-flags.sh @@ -65,5 +65,13 @@ if [ -e @out@/nix-support/cc-cflags-before ]; then NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="$(< @out@/nix-support/cc-cflags-before) $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@" fi +# Only add darwin min version flag if a default darwin min version is set, +# which is a signal that we're targetting darwin. +if [ "@darwinMinVersion@" ]; then + mangleVarSingle @darwinMinVersionVariable@ ${role_suffixes[@]+"${role_suffixes[@]}"} + + NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="-m@darwinPlatformForCC@-version-min=${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@" +fi + # That way forked processes will not extend these environment variables again. export NIX_CC_WRAPPER_FLAGS_SET_@suffixSalt@=1 diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh index 7e734f57773..aa25de33641 100644 --- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh @@ -25,11 +25,14 @@ dontLink=0 nonFlagArgs=0 cc1=0 # shellcheck disable=SC2193 -[[ "@prog@" = *++ ]] && isCpp=1 || isCpp=0 -cppInclude=1 +[[ "@prog@" = *++ ]] && isCxx=1 || isCxx=0 +cxxInclude=1 +cxxLibrary=1 cInclude=1 expandResponseParams "$@" +linkType=$(checkLinkType "$@") + declare -i n=0 nParams=${#params[@]} while (( "$n" < "$nParams" )); do @@ -49,15 +52,15 @@ while (( "$n" < "$nParams" )); do dontLink=1 elif [[ "$p" = -x && "$p2" = *-header ]]; then dontLink=1 - elif [[ "$p" = -x && "$p2" = c++* && "$isCpp" = 0 ]]; then - isCpp=1 + elif [[ "$p" = -x && "$p2" = c++* && "$isCxx" = 0 ]]; then + isCxx=1 elif [ "$p" = -nostdlib ]; then - isCpp=-1 + cxxLibrary=0 elif [ "$p" = -nostdinc ]; then cInclude=0 - cppInclude=0 + cxxInclude=0 elif [ "$p" = -nostdinc++ ]; then - cppInclude=0 + cxxInclude=0 elif [[ "$p" != -?* ]]; then # A dash alone signifies standard input; it is not a flag nonFlagArgs=1 @@ -128,11 +131,13 @@ if [ "$NIX_ENFORCE_NO_NATIVE_@suffixSalt@" = 1 ]; then params=(${rest+"${rest[@]}"}) fi -if [[ "$isCpp" = 1 ]]; then - if [[ "$cppInclude" = 1 ]]; then +if [[ "$isCxx" = 1 ]]; then + if [[ "$cxxInclude" = 1 ]]; then NIX_CFLAGS_COMPILE_@suffixSalt@+=" $NIX_CXXSTDLIB_COMPILE_@suffixSalt@" fi - NIX_CFLAGS_LINK_@suffixSalt@+=" $NIX_CXXSTDLIB_LINK_@suffixSalt@" + if [[ "$cxxLibrary" = 1 ]]; then + NIX_CFLAGS_LINK_@suffixSalt@+=" $NIX_CXXSTDLIB_LINK_@suffixSalt@" + fi fi source @out@/nix-support/add-hardening.sh @@ -145,21 +150,24 @@ if [ "$dontLink" != 1 ]; then # Add the flags that should only be passed to the compiler when # linking. - extraAfter+=($NIX_CFLAGS_LINK_@suffixSalt@) + extraAfter+=($(filterRpathFlags "$linkType" $NIX_CFLAGS_LINK_@suffixSalt@)) # Add the flags that should be passed to the linker (and prevent # `ld-wrapper' from adding NIX_LDFLAGS_@suffixSalt@ again). - for i in $NIX_LDFLAGS_BEFORE_@suffixSalt@; do + for i in $(filterRpathFlags "$linkType" $NIX_LDFLAGS_BEFORE_@suffixSalt@); do extraBefore+=("-Wl,$i") done - for i in $NIX_LDFLAGS_@suffixSalt@; do + if [[ "$linkType" == dynamic && -n "$NIX_DYNAMIC_LINKER_@suffixSalt@" ]]; then + extraBefore+=("-Wl,-dynamic-linker=$NIX_DYNAMIC_LINKER_@suffixSalt@") + fi + for i in $(filterRpathFlags "$linkType" $NIX_LDFLAGS_@suffixSalt@); do if [ "${i:0:3}" = -L/ ]; then extraAfter+=("$i") else extraAfter+=("-Wl,$i") fi done - export NIX_LDFLAGS_SET_@suffixSalt@=1 + export NIX_LINK_TYPE_@suffixSalt@=$linkType fi # As a very special hack, if the arguments are just `-v', then don't @@ -192,7 +200,15 @@ fi PATH="$path_backup" # Old bash workaround, see above. -exec @prog@ \ - ${extraBefore+"${extraBefore[@]}"} \ - ${params+"${params[@]}"} \ - ${extraAfter+"${extraAfter[@]}"} + +if (( "${NIX_CC_USE_RESPONSE_FILE:-@use_response_file_by_default@}" >= 1 )); then + exec @prog@ @<(printf "%q\n" \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"}) +else + exec @prog@ \ + ${extraBefore+"${extraBefore[@]}"} \ + ${params+"${params[@]}"} \ + ${extraAfter+"${extraAfter[@]}"} +fi diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 6ee287e287b..235d244a7c0 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -6,6 +6,7 @@ # compiler and the linker just "work". { name ? "" +, lib , stdenvNoCC , cc ? null, libc ? null, bintools, coreutils ? null, shell ? stdenvNoCC.shell , gccForLibs ? null @@ -18,7 +19,7 @@ , libcxx ? null }: -with stdenvNoCC.lib; +with lib; assert nativeTools -> !propagateDoc && nativePrefix != ""; assert !nativeTools -> @@ -34,11 +35,11 @@ let # # TODO(@Ericson2314) Make unconditional, or optional but always true by # default. - targetPrefix = stdenv.lib.optionalString (targetPlatform != hostPlatform) + targetPrefix = lib.optionalString (targetPlatform != hostPlatform) (targetPlatform.config + "-"); - ccVersion = stdenv.lib.getVersion cc; - ccName = stdenv.lib.removePrefix targetPrefix (stdenv.lib.getName cc); + ccVersion = lib.getVersion cc; + ccName = lib.removePrefix targetPrefix (lib.getName cc); libc_bin = if libc == null then null else getBin libc; libc_dev = if libc == null then null else getDev libc; @@ -57,29 +58,61 @@ let suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; expand-response-params = - if buildPackages.stdenv.hasCC && buildPackages.stdenv.cc != "/dev/null" + if (buildPackages.stdenv.hasCC or false) && buildPackages.stdenv.cc != "/dev/null" then import ../expand-response-params { inherit (buildPackages) stdenv; } else ""; + useGccForLibs = isClang + && libcxx == null + && !stdenv.targetPlatform.isDarwin + && !(stdenv.targetPlatform.useLLVM or false) + && !(stdenv.targetPlatform.useAndroidPrebuilt or false) + && !(stdenv.targetPlatform.isiOS or false) + && gccForLibs != null; + # older compilers (for example bootstrap's GCC 5) fail with -march=too-modern-cpu isGccArchSupported = arch: if isGNU then - { skylake = versionAtLeast ccVersion "6.0"; + { # Intel + skylake = versionAtLeast ccVersion "6.0"; skylake-avx512 = versionAtLeast ccVersion "6.0"; cannonlake = versionAtLeast ccVersion "8.0"; icelake-client = versionAtLeast ccVersion "8.0"; icelake-server = versionAtLeast ccVersion "8.0"; + cascadelake = versionAtLeast ccVersion "9.0"; + cooperlake = versionAtLeast ccVersion "10.0"; + tigerlake = versionAtLeast ccVersion "10.0"; knm = versionAtLeast ccVersion "8.0"; + # AMD + znver1 = versionAtLeast ccVersion "6.0"; + znver2 = versionAtLeast ccVersion "9.0"; + znver3 = versionAtLeast ccVersion "11.0"; }.${arch} or true else if isClang then - { cannonlake = versionAtLeast ccVersion "5.0"; + { # Intel + cannonlake = versionAtLeast ccVersion "5.0"; icelake-client = versionAtLeast ccVersion "7.0"; icelake-server = versionAtLeast ccVersion "7.0"; knm = versionAtLeast ccVersion "7.0"; + # AMD + znver1 = versionAtLeast ccVersion "4.0"; + znver2 = versionAtLeast ccVersion "9.0"; }.${arch} or true else false; + + darwinPlatformForCC = optionalString stdenv.targetPlatform.isDarwin ( + if (targetPlatform.darwinPlatform == "macos" && isGNU) then "macosx" + else targetPlatform.darwinPlatform + ); + + darwinMinVersion = optionalString stdenv.targetPlatform.isDarwin ( + stdenv.targetPlatform.darwinMinVersion + ); + + darwinMinVersionVariable = optionalString stdenv.targetPlatform.isDarwin + stdenv.targetPlatform.darwinMinVersionVariable; in # Ensure bintools matches @@ -102,6 +135,7 @@ stdenv.mkDerivation { gnugrep_bin = if nativeTools then "" else gnugrep; inherit targetPrefix suffixSalt; + inherit darwinPlatformForCC darwinMinVersion darwinMinVersionVariable; outputs = [ "out" ] ++ optionals propagateDoc [ "man" "info" ]; @@ -140,6 +174,7 @@ stdenv.mkDerivation { local dst="$1" local wrapper="$2" export prog="$3" + export use_response_file_by_default=${if isClang then "1" else "0"} substituteAll "$wrapper" "$out/bin/$dst" chmod +x "$out/bin/$dst" } @@ -228,8 +263,8 @@ stdenv.mkDerivation { setupHooks = [ ../setup-hooks/role.bash - ] ++ stdenv.lib.optional (cc.langC or true) ./setup-hook.sh - ++ stdenv.lib.optional (cc.langFortran or false) ./fortran-hook.sh; + ] ++ lib.optional (cc.langC or true) ./setup-hook.sh + ++ lib.optional (cc.langFortran or false) ./fortran-hook.sh; postFixup = # Ensure flags files exists, as some other programs cat them. (That these @@ -264,13 +299,28 @@ stdenv.mkDerivation { ## ## GCC libs for non-GCC support ## - + optionalString (isClang && libcxx == null && !(stdenv.targetPlatform.useLLVM or false) && gccForLibs != null) '' + + optionalString useGccForLibs '' echo "-B${gccForLibs}/lib/gcc/${targetPlatform.config}/${gccForLibs.version}" >> $out/nix-support/cc-cflags echo "-L${gccForLibs}/lib/gcc/${targetPlatform.config}/${gccForLibs.version}" >> $out/nix-support/cc-ldflags echo "-L${gccForLibs.lib}/${targetPlatform.config}/lib" >> $out/nix-support/cc-ldflags '' + # TODO We would like to connect this to `useGccForLibs`, but we cannot yet + # because `libcxxStdenv` on linux still needs this. Maybe someday we'll + # always set `useLLVM` on Darwin, and maybe also break down `useLLVM` into + # fine-grained use flags (libgcc vs compiler-rt, ld.lld vs legacy, libc++ + # vs libstdc++, etc.) since Darwin isn't `useLLVM` on all counts. (See + # https://clang.llvm.org/docs/Toolchain.html for all the axes one might + # break `useLLVM` into.) + + optionalString (isClang + && targetPlatform.isLinux + && !(stdenv.targetPlatform.useAndroidPrebuilt or false) + && !(stdenv.targetPlatform.useLLVM or false) + && gccForLibs != null) '' + echo "--gcc-toolchain=${gccForLibs}" >> $out/nix-support/cc-cflags + '' + ## ## General libc support ## @@ -308,11 +358,11 @@ stdenv.mkDerivation { # We have a libc++ directly, we have one via "smuggled" GCC, or we have one # bundled with the C compiler because it is GCC - + optionalString (libcxx != null || (isClang && !(stdenv.targetPlatform.useLLVM or false) && gccForLibs.langCC or false) || (isGNU && cc.langCC or false)) '' + + optionalString (libcxx != null || (useGccForLibs && gccForLibs.langCC or false) || (isGNU && cc.langCC or false)) '' touch "$out/nix-support/libcxx-cxxflags" touch "$out/nix-support/libcxx-ldflags" '' - + optionalString (libcxx == null && (isClang && !(stdenv.targetPlatform.useLLVM or false) && gccForLibs.langCC or false)) '' + + optionalString (libcxx == null && (useGccForLibs && gccForLibs.langCC or false)) '' for dir in ${gccForLibs}/include/c++/*; do echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags done @@ -321,9 +371,9 @@ stdenv.mkDerivation { done '' + optionalString (libcxx.isLLVM or false) ('' - echo "-isystem ${libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags + echo "-isystem ${lib.getDev libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags echo "-stdlib=libc++" >> $out/nix-support/libcxx-ldflags - '' + stdenv.lib.optionalString stdenv.targetPlatform.isLinux '' + '' + lib.optionalString stdenv.targetPlatform.isLinux '' echo "-lc++abi" >> $out/nix-support/libcxx-ldflags '') @@ -355,7 +405,7 @@ stdenv.mkDerivation { echo "$ccLDFlags" >> $out/nix-support/cc-ldflags echo "$ccCFlags" >> $out/nix-support/cc-cflags '' + optionalString (targetPlatform.isDarwin && (libcxx != null) && (cc.isClang or false)) '' - echo " -L${libcxx}/lib" >> $out/nix-support/cc-ldflags + echo " -L${lib.getLib libcxx}/lib" >> $out/nix-support/cc-ldflags '' ## @@ -383,32 +433,34 @@ stdenv.mkDerivation { # Always add -march based on cpu in triple. Sometimes there is a # discrepency (x86_64 vs. x86-64), so we provide an "arch" arg in # that case. - + optionalString ((targetPlatform ? platform.gcc.arch) && - isGccArchSupported targetPlatform.platform.gcc.arch) '' - echo "-march=${targetPlatform.platform.gcc.arch}" >> $out/nix-support/cc-cflags-before + # TODO: aarch64-darwin has mcpu incompatible with gcc + + optionalString ((targetPlatform ? gcc.arch) && (isClang || !(stdenv.isDarwin && stdenv.isAarch64)) && + isGccArchSupported targetPlatform.gcc.arch) '' + echo "-march=${targetPlatform.gcc.arch}" >> $out/nix-support/cc-cflags-before '' # -mcpu is not very useful. You should use mtune and march # instead. It’s provided here for backwards compatibility. - + optionalString (targetPlatform ? platform.gcc.cpu) '' - echo "-mcpu=${targetPlatform.platform.gcc.cpu}" >> $out/nix-support/cc-cflags-before + # TODO: aarch64-darwin has mcpu incompatible with gcc + + optionalString ((targetPlatform ? gcc.cpu) && (isClang || !(stdenv.isDarwin && stdenv.isAarch64))) '' + echo "-mcpu=${targetPlatform.gcc.cpu}" >> $out/nix-support/cc-cflags-before '' # -mfloat-abi only matters on arm32 but we set it here # unconditionally just in case. If the abi specifically sets hard # vs. soft floats we use it here. - + optionalString (targetPlatform ? platform.gcc.float-abi) '' - echo "-mfloat-abi=${targetPlatform.platform.gcc.float-abi}" >> $out/nix-support/cc-cflags-before + + optionalString (targetPlatform ? gcc.float-abi) '' + echo "-mfloat-abi=${targetPlatform.gcc.float-abi}" >> $out/nix-support/cc-cflags-before '' - + optionalString (targetPlatform ? platform.gcc.fpu) '' - echo "-mfpu=${targetPlatform.platform.gcc.fpu}" >> $out/nix-support/cc-cflags-before + + optionalString (targetPlatform ? gcc.fpu) '' + echo "-mfpu=${targetPlatform.gcc.fpu}" >> $out/nix-support/cc-cflags-before '' - + optionalString (targetPlatform ? platform.gcc.mode) '' - echo "-mmode=${targetPlatform.platform.gcc.mode}" >> $out/nix-support/cc-cflags-before + + optionalString (targetPlatform ? gcc.mode) '' + echo "-mmode=${targetPlatform.gcc.mode}" >> $out/nix-support/cc-cflags-before '' - + optionalString (targetPlatform ? platform.gcc.tune && - isGccArchSupported targetPlatform.platform.gcc.tune) '' - echo "-mtune=${targetPlatform.platform.gcc.tune}" >> $out/nix-support/cc-cflags-before + + optionalString (targetPlatform ? gcc.tune && + isGccArchSupported targetPlatform.gcc.tune) '' + echo "-mtune=${targetPlatform.gcc.tune}" >> $out/nix-support/cc-cflags-before '' # TODO: categorize these and figure out a better place for them @@ -420,10 +472,12 @@ stdenv.mkDerivation { hardening_unsupported_flags+=" stackprotector pic" '' + optionalString (targetPlatform.libc == "newlib") '' hardening_unsupported_flags+=" stackprotector fortify pie pic" + '' + optionalString (targetPlatform.libc == "musl" && targetPlatform.isx86_32) '' + hardening_unsupported_flags+=" stackprotector" '' + optionalString targetPlatform.isNetBSD '' hardening_unsupported_flags+=" stackprotector fortify" '' + optionalString cc.langAda or false '' - hardening_unsupported_flags+=" stackprotector strictoverflow" + hardening_unsupported_flags+=" format stackprotector strictoverflow" '' + optionalString cc.langD or false '' hardening_unsupported_flags+=" format" '' + optionalString targetPlatform.isWasm '' @@ -436,6 +490,14 @@ stdenv.mkDerivation { done '' + + optionalString stdenv.targetPlatform.isDarwin '' + echo "-arch ${targetPlatform.darwinArch}" >> $out/nix-support/cc-cflags + '' + + + optionalString targetPlatform.isAndroid '' + echo "-D__ANDROID_API__=${targetPlatform.sdkVer}" >> $out/nix-support/cc-cflags + '' + # There are a few tools (to name one libstdcxx5) which do not work # well with multi line flags, so make the flags single line again + '' @@ -462,7 +524,7 @@ stdenv.mkDerivation { let cc_ = if cc != null then cc else {}; in (if cc_ ? meta then removeAttrs cc.meta ["priority"] else {}) // { description = - stdenv.lib.attrByPath ["meta" "description"] "System C compiler" cc_ + lib.attrByPath ["meta" "description"] "System C compiler" cc_ + " (wrapper script)"; priority = 10; }; diff --git a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh index d3f7d382c19..5714b228c59 100644 --- a/pkgs/build-support/cc-wrapper/gnat-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/gnat-wrapper.sh @@ -15,6 +15,8 @@ if [[ -n "@coreutils_bin@" && -n "@gnugrep_bin@" ]]; then PATH="@coreutils_bin@/bin:@gnugrep_bin@/bin" fi +cInclude=0 + source @out@/nix-support/utils.bash # Flirting with a layer violation here. diff --git a/pkgs/build-support/coq/default.nix b/pkgs/build-support/coq/default.nix new file mode 100644 index 00000000000..ba300f2f8cf --- /dev/null +++ b/pkgs/build-support/coq/default.nix @@ -0,0 +1,109 @@ +{ lib, stdenv, coqPackages, coq, fetchzip }@args: +let lib = import ./extra-lib.nix {inherit (args) lib;}; in +with builtins; with lib; +let + isGitHubDomain = d: match "^github.*" d != null; + isGitLabDomain = d: match "^gitlab.*" d != null; +in +{ pname, + version ? null, + fetcher ? null, + owner ? "coq-community", + domain ? "github.com", + repo ? pname, + defaultVersion ? null, + releaseRev ? (v: v), + displayVersion ? {}, + release ? {}, + extraBuildInputs ? [], + namePrefix ? [], + enableParallelBuilding ? true, + extraInstallFlags ? [], + setCOQBIN ? true, + mlPlugin ? false, + useMelquiondRemake ? null, + dropAttrs ? [], + keepAttrs ? [], + dropDerivationAttrs ? [], + useDune2ifVersion ? (x: false), + useDune2 ? false, + ... +}@args: +let + args-to-remove = foldl (flip remove) ([ + "version" "fetcher" "repo" "owner" "domain" "releaseRev" + "displayVersion" "defaultVersion" "useMelquiondRemake" + "release" "extraBuildInputs" "extraPropagatedBuildInputs" "namePrefix" + "meta" "useDune2ifVersion" "useDune2" + "extraInstallFlags" "setCOQBIN" "mlPlugin" + "dropAttrs" "dropDerivationAttrs" "keepAttrs" ] ++ dropAttrs) keepAttrs; + fetch = import ../coq/meta-fetch/default.nix + { inherit lib stdenv fetchzip; } ({ + inherit release releaseRev; + location = { inherit domain owner repo; }; + } // optionalAttrs (args?fetcher) {inherit fetcher;}); + fetched = fetch (if !isNull version then version else defaultVersion); + namePrefix = args.namePrefix or [ "coq" ]; + display-pkg = n: sep: v: + let d = displayVersion.${n} or (if sep == "" then ".." else true); in + n + optionalString (v != "" && v != null) (switch d [ + { case = true; out = sep + v; } + { case = "."; out = sep + versions.major v; } + { case = ".."; out = sep + versions.majorMinor v; } + { case = "..."; out = sep + versions.majorMinorPatch v; } + { case = isFunction; out = optionalString (d v != "") (sep + d v); } + { case = isString; out = optionalString (d != "") (sep + d); } + ] "") + optionalString (v == null) "-broken"; + append-version = p: n: p + display-pkg n "" coqPackages.${n}.version + "-"; + prefix-name = foldl append-version "" namePrefix; + var-coqlib-install = (optionalString (versions.isGe "8.7" coq.coq-version) "COQMF_") + "COQLIB"; + useDune2 = args.useDune2 or (useDune2ifVersion fetched.version); +in + +stdenv.mkDerivation (removeAttrs ({ + + name = prefix-name + (display-pkg pname "-" fetched.version); + + inherit (fetched) version src; + + buildInputs = [ coq ] + ++ optionals mlPlugin coq.ocamlBuildInputs + ++ optionals useDune2 [coq.ocaml coq.ocamlPackages.dune_2] + ++ extraBuildInputs; + inherit enableParallelBuilding; + + meta = ({ platforms = coq.meta.platforms; } // + (switch domain [{ + case = pred.union isGitHubDomain isGitLabDomain; + out = { homepage = "https://${domain}/${owner}/${repo}"; }; + }] {}) // + optionalAttrs (fetched.broken or false) { coqFilter = true; broken = true; }) // + (args.meta or {}) ; + +} +// (optionalAttrs setCOQBIN { COQBIN = "${coq}/bin/"; }) +// (optionalAttrs (!args?installPhase && !args?useMelquiondRemake) { + installFlags = + [ "${var-coqlib-install}=$(out)/lib/coq/${coq.coq-version}/" ] ++ + optional (match ".*doc$" (args.installTargets or "") != null) + "DOCDIR=$(out)/share/coq/${coq.coq-version}/" ++ + extraInstallFlags; +}) +// (optionalAttrs useDune2 { + installPhase = '' + runHook preInstall + dune install --prefix=$out + mv $out/lib/coq $out/lib/TEMPORARY + mkdir $out/lib/coq/ + mv $out/lib/TEMPORARY $out/lib/coq/${coq.coq-version} + runHook postInstall + ''; +}) +// (optionalAttrs (args?useMelquiondRemake) rec { + COQUSERCONTRIB = "$out/lib/coq/${coq.coq-version}/user-contrib"; + preConfigurePhases = "autoconf"; + configureFlags = [ "--libdir=${COQUSERCONTRIB}/${useMelquiondRemake.logpath or ""}" ]; + buildPhase = "./remake -j$NIX_BUILD_CORES"; + installPhase = "./remake install"; +}) +// (removeAttrs args args-to-remove)) dropDerivationAttrs) diff --git a/pkgs/build-support/coq/extra-lib.nix b/pkgs/build-support/coq/extra-lib.nix new file mode 100644 index 00000000000..65b48f51126 --- /dev/null +++ b/pkgs/build-support/coq/extra-lib.nix @@ -0,0 +1,145 @@ +{ lib }: +with builtins; with lib; recursiveUpdate lib (rec { + + versions = + let + truncate = n: v: concatStringsSep "." (take n (splitVersion v)); + opTruncate = op: v0: v: let n = length (splitVersion v0); in + op (truncate n v) (truncate n v0); + in rec { + + /* Get string of the first n parts of a version string. + + Example: + - truncate 2 "1.2.3-stuff" + => "1.2" + + - truncate 4 "1.2.3-stuff" + => "1.2.3.stuff" + */ + + inherit truncate; + + /* Get string of the first three parts (major, minor and patch) + of a version string. + + Example: + majorMinorPatch "1.2.3-stuff" + => "1.2.3" + */ + majorMinorPatch = truncate 3; + + /* Version comparison predicates, + - isGe v0 v <-> v is greater or equal than v0 [*] + - isLe v0 v <-> v is lesser or equal than v0 [*] + - isGt v0 v <-> v is strictly greater than v0 [*] + - isLt v0 v <-> v is strictly lesser than v0 [*] + - isEq v0 v <-> v is equal to v0 [*] + - range low high v <-> v is between low and high [**] + + [*] truncating v to the same number of digits as v0 + [**] truncating v to low for the lower bound and high for the upper bound + + Examples: + - isGe "8.10" "8.10.1" + => true + - isLe "8.10" "8.10.1" + => true + - isGt "8.10" "8.10.1" + => false + - isGt "8.10.0" "8.10.1" + => true + - isEq "8.10" "8.10.1" + => true + - range "8.10" "8.11" "8.11.1" + => true + - range "8.10" "8.11+" "8.11.0" + => false + - range "8.10" "8.11+" "8.11+beta1" + => false + + */ + isGe = opTruncate versionAtLeast; + isGt = opTruncate (flip versionOlder); + isLe = opTruncate (flip versionAtLeast); + isLt = opTruncate versionOlder; + isEq = opTruncate pred.equal; + range = low: high: pred.inter (versions.isGe low) (versions.isLe high); + }; + + /* Returns a list of list, splitting it using a predicate. + This is analoguous to builtins.split sep list, + with a predicate as a separator and a list instead of a string. + + Type: splitList :: (a -> bool) -> [a] -> [[a]] + + Example: + splitList (x: x == "x") [ "y" "x" "z" "t" ] + => [ [ "y" ] "x" [ "z" "t" ] ] + */ + splitList = pred: l: # put in file lists + let loop = (vv: v: l: if l == [] then vv ++ [v] + else let hd = head l; tl = tail l; in + if pred hd then loop (vv ++ [ v hd ]) [] tl else loop vv (v ++ [hd]) tl); + in loop [] [] l; + + pred = { + /* Predicate intersection, union, and complement */ + inter = p: q: x: p x && q x; + union = p: q: x: p x || q x; + compl = p: x: ! p x; + true = p: true; + false = p: false; + + /* predicate "being equal to y" */ + equal = y: x: x == y; + }; + + /* Emulate a "switch - case" construct, + instead of relying on `if then else if ...` */ + /* Usage: + ```nix + switch-if [ + if-clause-1 + .. + if-clause-k + ] default-out + ``` + where a if-clause has the form `{ cond = b; out = r; }` + the first branch such as `b` is true */ + + switch-if = c: d: (findFirst (getAttr "cond") {} c).out or d; + + /* Usage: + ```nix + switch x [ + simple-clause-1 + .. + simple-clause-k + ] default-out + ``` + where a simple-clause has the form `{ case = p; out = r; }` + the first branch such as `p x` is true + or + ```nix + switch [ x1 .. xn ] [ + complex-clause-1 + .. + complex-clause-k + ] default-out + ``` + where a complex-clause is either a simple-clause + or has the form { cases = [ p1 .. pn ]; out = r; } + in which case the first branch such as all `pi x` are true + + if the variables p are not functions, + they are converted to a equal p + if out is missing the default-out is taken */ + + switch = var: clauses: default: with pred; let + compare = f: if isFunction f then f else equal f; + combine = cl: var: + if cl?case then compare cl.case var + else all (equal true) (zipListsWith compare cl.cases var); in + switch-if (map (cl: { cond = combine cl var; inherit (cl) out; }) clauses) default; +}) diff --git a/pkgs/build-support/coq/meta-fetch/default.nix b/pkgs/build-support/coq/meta-fetch/default.nix new file mode 100644 index 00000000000..e7b15af4f06 --- /dev/null +++ b/pkgs/build-support/coq/meta-fetch/default.nix @@ -0,0 +1,70 @@ +{ lib, stdenv, fetchzip }@args: +let lib' = lib; in +let lib = import ../extra-lib.nix {lib = lib';}; in +with builtins; with lib; +let + default-fetcher = {domain ? "github.com", owner ? "", repo, rev, name ? "source", sha256 ? null, ...}@args: + let ext = if args?sha256 then "zip" else "tar.gz"; + fmt = if args?sha256 then "zip" else "tarball"; + pr = match "^#(.*)$" rev; + url = switch-if [ + { cond = isNull pr && !isNull (match "^github.*" domain); + out = "https://${domain}/${owner}/${repo}/archive/${rev}.${ext}"; } + { cond = !isNull pr && !isNull (match "^github.*" domain); + out = "https://api.${domain}/repos/${owner}/${repo}/${fmt}/pull/${head pr}/head"; } + { cond = isNull pr && !isNull (match "^gitlab.*" domain); + out = "https://${domain}/${owner}/${repo}/-/archive/${rev}/${repo}-${rev}.${ext}"; } + { cond = !isNull (match "(www.)?mpi-sws.org" domain); + out = "https://www.mpi-sws.org/~${owner}/${repo}/download/${repo}-${rev}.${ext}";} + ] (throw "meta-fetch: no fetcher found for domain ${domain} on ${rev}"); + fetch = x: if args?sha256 then fetchzip (x // { inherit sha256; }) else fetchTarball x; + in fetch { inherit url ; }; +in +{ + fetcher ? default-fetcher, + location, + release ? {}, + releaseRev ? (v: v), +}: +let isVersion = x: isString x && match "^/.*" x == null && release?${x}; + shortVersion = x: if (isString x && match "^/.*" x == null) + then findFirst (v: versions.majorMinor v == x) null + (sort versionAtLeast (attrNames release)) + else null; + isShortVersion = x: shortVersion x != null; + isPathString = x: isString x && match "^/.*" x != null && pathExists x; in +arg: +switch arg [ + { case = isNull; out = { version = "broken"; src = ""; broken = true; }; } + { case = isPathString; out = { version = "dev"; src = arg; }; } + { case = pred.union isVersion isShortVersion; + out = let v = if isVersion arg then arg else shortVersion arg; in + let + given-sha256 = release.${v}.sha256 or ""; + sha256 = if given-sha256 == "" then lib.fakeSha256 else given-sha256; + rv = release.${v} // { inherit sha256; }; in + { + version = rv.version or v; + src = rv.src or fetcher (location // { rev = releaseRev v; } // rv); + }; + } + { case = isString; + out = let + splitted = filter isString (split ":" arg); + rev = last splitted; + has-owner = length splitted > 1; + version = "dev"; in { + inherit version; + src = fetcher (location // { inherit rev; } // + (optionalAttrs has-owner { owner = head splitted; })); + }; } + { case = isAttrs; + out = let + { version = arg.version or "dev"; + src = (arg.fetcher or fetcher) (location // (arg.location or {})); + }; } + { case = isPath; + out = { + version = "dev" ; + src = builtins.path {path = arg; name = location.name or "source";}; }; } +] (throw "not a valid source description") diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index bf815af6f7c..4bda09670ab 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -1,4 +1,5 @@ { + bashInteractive, buildPackages, cacert, callPackage, @@ -6,6 +7,7 @@ coreutils, docker, e2fsprogs, + fakeroot, findutils, go, jq, @@ -15,27 +17,30 @@ moreutils, nix, pigz, - referencesByPopularity, rsync, runCommand, runtimeShell, shadow, skopeo, - stdenv, storeDir ? builtins.storeDir, substituteAll, symlinkJoin, - utillinux, + util-linux, vmTools, writeReferencesToFile, writeScript, writeText, + writeTextDir, writePython3, + system, # Note: This is the cross system we're compiling for }: -# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future. let + inherit (lib) + optionals + ; + mkDbExtraCommand = contents: let contentsList = if builtins.isList contents then contents else [ contents ]; in '' @@ -48,7 +53,7 @@ let # A user is required by nix # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478 export USER=nobody - ${nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration + ${buildPackages.nix}/bin/nix-store --load-db < ${closureInfo {rootPaths = contentsList;}}/registration mkdir -p nix/var/nix/gcroots/docker/ for i in ${lib.concatStringsSep " " contentsList}; do @@ -56,11 +61,18 @@ let done; ''; + # The OCI Image specification recommends that configurations use values listed + # in the Go Language document for GOARCH. + # Reference: https://github.com/opencontainers/image-spec/blob/master/config.md#properties + # For the mapping from Nixpkgs system parameters to GOARCH, we can reuse the + # mapping from the go package. + defaultArch = go.GOARCH; + in rec { examples = callPackage ./examples.nix { - inherit buildImage pullImage shadowSetup buildImageWithNixDb; + inherit buildImage buildLayeredImage fakeNss pullImage shadowSetup buildImageWithNixDb; }; pullImage = let @@ -72,12 +84,14 @@ rec { , imageDigest , sha256 , os ? "linux" - , arch ? buildPackages.go.GOARCH + , arch ? defaultArch # This is used to set name to the pulled image , finalImageName ? imageName # This used to set a tag to the pulled image , finalImageTag ? "latest" + # This is used to disable TLS certificate verification, allowing access to http registries on (hopefully) trusted networks + , tlsVerify ? true , name ? fixName "docker-image-${finalImageName}-${finalImageTag}.tar" }: @@ -86,7 +100,7 @@ rec { inherit imageDigest; imageName = finalImageName; imageTag = finalImageTag; - impureEnvVars = stdenv.lib.fetchers.proxyImpureEnvVars; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; outputHashMode = "flat"; outputHashAlgo = "sha256"; outputHash = sha256; @@ -97,11 +111,18 @@ rec { sourceURL = "docker://${imageName}@${imageDigest}"; destNameTag = "${finalImageName}:${finalImageTag}"; } '' - skopeo --insecure-policy --tmpdir=$TMPDIR --override-os ${os} --override-arch ${arch} copy "$sourceURL" "docker-archive://$out:$destNameTag" + skopeo \ + --src-tls-verify=${lib.boolToString tlsVerify} \ + --insecure-policy \ + --tmpdir=$TMPDIR \ + --override-os ${os} \ + --override-arch ${arch} \ + copy "$sourceURL" "docker-archive://$out:$destNameTag" \ + | cat # pipe through cat to force-disable progress bar ''; # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash. - # And we cannot untar it, because then we cannot preserve permissions ecc. + # And we cannot untar it, because then we cannot preserve permissions etc. tarsum = runCommand "tarsum" { nativeBuildInputs = [ go ]; } '' @@ -111,8 +132,9 @@ rec { cp ${./tarsum.go} tarsum.go export GOPATH=$(pwd) export GOCACHE="$TMPDIR/go-cache" + export GO111MODULE=off mkdir -p src/github.com/docker/docker/pkg - ln -sT ${docker.src}/components/engine/pkg/tarsum src/github.com/docker/docker/pkg/tarsum + ln -sT ${docker.moby-src}/pkg/tarsum src/github.com/docker/docker/pkg/tarsum go build mkdir -p $out/bin @@ -194,7 +216,7 @@ rec { }; inherit fromImage fromImageName fromImageTag; - nativeBuildInputs = [ utillinux e2fsprogs jshon rsync jq ]; + nativeBuildInputs = [ util-linux e2fsprogs jshon rsync jq ]; } '' mkdir disk mkfs /dev/${vmTools.hd} @@ -340,7 +362,7 @@ rec { # Tar up the layer and throw it into 'layer.tar'. echo "Packing layer..." mkdir $out - tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum) + tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee -p $out/layer.tar | tarsum) # Add a 'checksum' field to the JSON, with the value set to the # checksum of the tarball. @@ -410,7 +432,11 @@ rec { # details on what's going on here; basically this command # means that the runAsRootScript will be executed in a nearly # completely isolated environment. - unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript} + # + # Ideally we would use --mount-proc=mnt/proc or similar, but this + # doesn't work. The workaround is to setup proc after unshare. + # See: https://github.com/karelzak/util-linux/issues/648 + unshare -imnpuf --mount-proc sh -c 'mount --rbind /proc mnt/proc && chroot mnt ${runAsRootScript}' # Unmount directories and remove them. umount -R mnt/dev mnt/sys mnt${storeDir} @@ -425,7 +451,7 @@ rec { echo "Packing layer..." mkdir -p $out tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf - . | - tee $out/layer.tar | + tee -p $out/layer.tar | ${tarsum}/bin/tarsum) cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json @@ -440,10 +466,10 @@ rec { let stream = streamLayeredImage args; in - runCommand "${name}.tar.gz" { + runCommand "${baseNameOf name}.tar.gz" { inherit (stream) imageName; passthru = { inherit (stream) imageTag; }; - buildInputs = [ pigz ]; + nativeBuildInputs = [ pigz ]; } "${stream} | pigz -nT > $out"; # 1. extract the base image @@ -488,7 +514,7 @@ rec { baseJson = let pure = writeText "${baseName}-config.json" (builtins.toJSON { inherit created config; - architecture = buildPackages.go.GOARCH; + architecture = defaultArch; os = "linux"; }); impure = runCommand "${baseName}-config.json" @@ -511,16 +537,16 @@ rec { }; result = runCommand "docker-image-${baseName}.tar.gz" { nativeBuildInputs = [ jshon pigz coreutils findutils jq moreutils ]; - # Image name and tag must be lowercase + # Image name must be lowercase imageName = lib.toLower name; - imageTag = if tag == null then "" else lib.toLower tag; + imageTag = if tag == null then "" else tag; inherit fromImage baseJson; layerClosure = writeReferencesToFile layer; passthru.buildArgs = args; passthru.layer = layer; passthru.imageTag = if tag != null - then lib.toLower tag + then tag else lib.head (lib.strings.splitString "-" (baseNameOf result.outPath)); # Docker can't be made to run darwin binaries @@ -674,6 +700,69 @@ rec { in result; + # Merge the tarballs of images built with buildImage into a single + # tarball that contains all images. Running `docker load` on the resulting + # tarball will load the images into the docker daemon. + mergeImages = images: runCommand "merge-docker-images" + { + inherit images; + nativeBuildInputs = [ pigz jq ]; + } '' + mkdir image inputs + # Extract images + repos=() + manifests=() + for item in $images; do + name=$(basename $item) + mkdir inputs/$name + tar -I pigz -xf $item -C inputs/$name + if [ -f inputs/$name/repositories ]; then + repos+=(inputs/$name/repositories) + fi + if [ -f inputs/$name/manifest.json ]; then + manifests+=(inputs/$name/manifest.json) + fi + done + # Copy all layers from input images to output image directory + cp -R --no-clobber inputs/*/* image/ + # Merge repositories objects and manifests + jq -s add "''${repos[@]}" > repositories + jq -s add "''${manifests[@]}" > manifest.json + # Replace output image repositories and manifest with merged versions + mv repositories image/repositories + mv manifest.json image/manifest.json + # Create tarball and gzip + tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nT > $out + ''; + + + # Provide a /etc/passwd and /etc/group that contain root and nobody. + # Useful when packaging binaries that insist on using nss to look up + # username/groups (like nginx). + # /bin/sh is fine to not exist, and provided by another shim. + fakeNss = symlinkJoin { + name = "fake-nss"; + paths = [ + (writeTextDir "etc/passwd" '' + root:x:0:0:root user:/var/empty:/bin/sh + nobody:x:65534:65534:nobody:/var/empty:/bin/sh + '') + (writeTextDir "etc/group" '' + root:x:0: + nobody:x:65534: + '') + (runCommand "var-empty" {} '' + mkdir -p $out/var/empty + '') + ]; + }; + + # This provides /bin/sh, pointing to bashInteractive. + binSh = runCommand "bin-sh" {} '' + mkdir -p $out/bin + ln -s ${bashInteractive}/bin/bash $out/bin/sh + ''; + # Build an image and populate its nix database with the provided # contents. The main purpose is to be able to use nix commands in # the container. @@ -695,6 +784,8 @@ rec { name, # Image tag, the Nix's output hash will be used if null tag ? null, + # Parent image, to append to. + fromImage ? null, # Files to put on the image (a nix store path or list of paths). contents ? [], # Docker config; e.g. what command to run on the container. @@ -704,56 +795,83 @@ rec { created ? "1970-01-01T00:00:01Z", # Optional bash script to run on the files prior to fixturizing the layer. extraCommands ? "", + # Optional bash script to run inside fakeroot environment. + # Could be used for changing ownership of files in customisation layer. + fakeRootCommands ? "", # We pick 100 to ensure there is plenty of room for extension. I # believe the actual maximum is 128. - maxLayers ? 100 + maxLayers ? 100, + # Whether to include store paths in the image. You generally want to leave + # this on, but tooling may disable this to insert the store paths more + # efficiently via other means, such as bind mounting the host store. + includeStorePaths ? true, }: assert (lib.assertMsg (maxLayers > 1) "the maxLayers argument of dockerTools.buildLayeredImage function must be greather than 1 (current value: ${toString maxLayers})"); let + baseName = baseNameOf name; + streamScript = writePython3 "stream" {} ./stream_layered_image.py; - baseJson = writeText "${name}-base.json" (builtins.toJSON { + baseJson = writeText "${baseName}-base.json" (builtins.toJSON { inherit config; - architecture = buildPackages.go.GOARCH; + architecture = defaultArch; os = "linux"; }); - customisationLayer = runCommand "${name}-customisation-layer" { inherit extraCommands; } '' - cp -r ${contentsEnv}/ $out - if [[ -n $extraCommands ]]; then - chmod u+w $out - (cd $out; eval "$extraCommands") - fi - ''; - contentsEnv = symlinkJoin { - name = "${name}-bulk-layers"; - paths = if builtins.isList contents - then contents - else [ contents ]; + contentsList = if builtins.isList contents then contents else [ contents ]; + + # We store the customisation layer as a tarball, to make sure that + # things like permissions set on 'extraCommands' are not overriden + # by Nix. Then we precompute the sha256 for performance. + customisationLayer = symlinkJoin { + name = "${baseName}-customisation-layer"; + paths = contentsList; + inherit extraCommands fakeRootCommands; + nativeBuildInputs = [ fakeroot ]; + postBuild = '' + mv $out old_out + (cd old_out; eval "$extraCommands" ) + + mkdir $out + + fakeroot bash -c ' + source $stdenv/setup + cd old_out + eval "$fakeRootCommands" + tar \ + --sort name \ + --numeric-owner --mtime "@$SOURCE_DATE_EPOCH" \ + --hard-dereference \ + -cf $out/layer.tar . + ' + + sha256sum $out/layer.tar \ + | cut -f 1 -d ' ' \ + > $out/checksum + ''; }; - # NOTE: the `closures` parameter is a list of closures to include. - # The TOP LEVEL store paths themselves will never be present in the - # resulting image. At this time (2020-06-18) none of these layers - # are appropriate to include, as they are all created as - # implementation details of dockerTools. - closures = [ baseJson contentsEnv ]; - overallClosure = writeText "closure" (lib.concatStringsSep " " closures); - conf = runCommand "${name}-conf.json" { - inherit maxLayers created; + closureRoots = optionals includeStorePaths /* normally true */ ( + [ baseJson ] ++ contentsList + ); + overallClosure = writeText "closure" (lib.concatStringsSep " " closureRoots); + + # These derivations are only created as implementation details of docker-tools, + # so they'll be excluded from the created images. + unnecessaryDrvs = [ baseJson overallClosure ]; + + conf = runCommand "${baseName}-conf.json" { + inherit fromImage maxLayers created; imageName = lib.toLower name; passthru.imageTag = if tag != null then tag else lib.head (lib.strings.splitString "-" (baseNameOf conf.outPath)); - paths = referencesByPopularity overallClosure; - buildInputs = [ jq ]; + paths = buildPackages.referencesByPopularity overallClosure; + nativeBuildInputs = [ jq ]; } '' - paths() { - cat $paths ${lib.concatMapStringsSep " " (path: "| (grep -v ${path} || true)") (closures ++ [ overallClosure ])} - } ${if (tag == null) then '' outName="$(basename "$out")" outHash=$(echo "$outName" | cut -d - -f 1) @@ -768,6 +886,33 @@ rec { created="$(date -Iseconds -d "$created")" fi + paths() { + cat $paths ${lib.concatMapStringsSep " " + (path: "| (grep -v ${path} || true)") + unnecessaryDrvs} + } + + # Compute the number of layers that are already used by a potential + # 'fromImage' as well as the customization layer. Ensure that there is + # still at least one layer available to store the image contents. + usedLayers=0 + + # subtract number of base image layers + if [[ -n "$fromImage" ]]; then + (( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') )) + fi + + # one layer will be taken up by the customisation layer + (( usedLayers += 1 )) + + if ! (( $usedLayers < $maxLayers )); then + echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \ + "'extraCommands', but only maxLayers=$maxLayers were" \ + "allowed. At least 1 layer is required to store contents." + exit 1 + fi + availableLayers=$(( maxLayers - usedLayers )) + # Create $maxLayers worth of Docker Layers, one layer per store path # unless there are more paths than $maxLayers. In that case, create # $maxLayers-1 for the most popular layers, and smush the remainaing @@ -785,26 +930,36 @@ rec { | (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ] | map(select(length > 0)) ' \ - --argjson maxLayers "$(( maxLayers - 1 ))" # one layer will be taken up by the customisation layer + --argjson maxLayers "$availableLayers" )" cat ${baseJson} | jq ' . + { + "store_dir": $store_dir, + "from_image": $from_image, "store_layers": $store_layers, "customisation_layer", $customisation_layer, "repo_tag": $repo_tag, "created": $created } - ' --argjson store_layers "$store_layers" \ + ' --arg store_dir "${storeDir}" \ + --argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \ + --argjson store_layers "$store_layers" \ --arg customisation_layer ${customisationLayer} \ --arg repo_tag "$imageName:$imageTag" \ --arg created "$created" | tee $out ''; - result = runCommand "stream-${name}" { + result = runCommand "stream-${baseName}" { inherit (conf) imageName; - passthru = { inherit (conf) imageTag; }; - buildInputs = [ makeWrapper ]; + passthru = { + inherit (conf) imageTag; + + # Distinguish tarballs and exes at the Nix level so functions that + # take images can know in advance how the image is supposed to be used. + isExe = true; + }; + nativeBuildInputs = [ makeWrapper ]; } '' makeWrapper ${streamScript} $out --add-flags ${conf} ''; diff --git a/pkgs/build-support/docker/detjson.py b/pkgs/build-support/docker/detjson.py index 439c2131387..fe82cbea11b 100644 --- a/pkgs/build-support/docker/detjson.py +++ b/pkgs/build-support/docker/detjson.py @@ -37,4 +37,4 @@ def main(): json.dump(j, sys.stdout, sort_keys=True) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index bc107471762..f890d0a77a2 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -7,7 +7,7 @@ # $ nix-build '<nixpkgs>' -A dockerTools.examples.redis # $ docker load < result -{ pkgs, buildImage, pullImage, shadowSetup, buildImageWithNixDb }: +{ pkgs, buildImage, buildLayeredImage, fakeNss, pullImage, shadowSetup, buildImageWithNixDb, pkgsCross }: rec { # 1. basic example @@ -44,7 +44,7 @@ rec { nginx = let nginxPort = "80"; nginxConf = pkgs.writeText "nginx.conf" '' - user nginx nginx; + user nobody nobody; daemon off; error_log /dev/stdout info; pid /dev/null; @@ -64,10 +64,13 @@ rec { <html><body><h1>Hello from NGINX</h1></body></html> ''; in - buildImage { + buildLayeredImage { name = "nginx-container"; tag = "latest"; - contents = pkgs.nginx; + contents = [ + fakeNss + pkgs.nginx + ]; extraCommands = '' # nginx still tries to read this directory even if error_log @@ -75,12 +78,6 @@ rec { mkdir -p var/log/nginx mkdir -p var/cache/nginx ''; - runAsRoot = '' - #!${pkgs.stdenv.shell} - ${shadowSetup} - groupadd --system nginx - useradd --system --gid nginx nginx - ''; config = { Cmd = [ "nginx" "-c" nginxConf ]; @@ -94,7 +91,7 @@ rec { nixFromDockerHub = pullImage { imageName = "nixos/nix"; imageDigest = "sha256:85299d86263a3059cf19f419f9d286cc9f06d3c13146a8ebbb21b3437f598357"; - sha256 = "07q9y9r7fsd18sy95ybrvclpkhlal12d30ybnf089hq7v1hgxbi7"; + sha256 = "19fw0n3wmddahzr20mhdqv6jkjn1kanh6n2mrr08ai53dr8ph5n7"; finalImageTag = "2.2.1"; finalImageName = "nix"; }; @@ -191,7 +188,25 @@ rec { }; }; - # 12. example of running something as root on top of a parent image + # 12 Create a layered image on top of a layered image + layered-on-top-layered = pkgs.dockerTools.buildLayeredImage { + name = "layered-on-top-layered"; + tag = "latest"; + fromImage = layered-image; + extraCommands = '' + mkdir ./example-output + chmod 777 ./example-output + ''; + config = { + Env = [ "PATH=${pkgs.coreutils}/bin/" ]; + WorkingDir = "/example-output"; + Cmd = [ + "${pkgs.bash}/bin/bash" "-c" "echo hello > foo; cat foo" + ]; + }; + }; + + # 13. example of running something as root on top of a parent image # Regression test related to PR #52109 runAsRootParentImage = buildImage { name = "runAsRootParentImage"; @@ -200,7 +215,7 @@ rec { fromImage = bash; }; - # 13. example of 3 layers images This image is used to verify the + # 14. example of 3 layers images This image is used to verify the # order of layers is correct. # It allows to validate # - the layer of parent are below @@ -238,23 +253,36 @@ rec { ''; }; - # 14. Environment variable inheritance. + # 15. Environment variable inheritance. # Child image should inherit parents environment variables, # optionally overriding them. - environmentVariables = let - parent = pkgs.dockerTools.buildImage { - name = "parent"; - tag = "latest"; - config = { - Env = [ - "FROM_PARENT=true" - "LAST_LAYER=parent" - ]; - }; + environmentVariablesParent = pkgs.dockerTools.buildImage { + name = "parent"; + tag = "latest"; + config = { + Env = [ + "FROM_PARENT=true" + "LAST_LAYER=parent" + ]; }; - in pkgs.dockerTools.buildImage { + }; + + environmentVariables = pkgs.dockerTools.buildImage { + name = "child"; + fromImage = environmentVariablesParent; + tag = "latest"; + contents = [ pkgs.coreutils ]; + config = { + Env = [ + "FROM_CHILD=true" + "LAST_LAYER=child" + ]; + }; + }; + + environmentVariablesLayered = pkgs.dockerTools.buildLayeredImage { name = "child"; - fromImage = parent; + fromImage = environmentVariablesParent; tag = "latest"; contents = [ pkgs.coreutils ]; config = { @@ -265,14 +293,14 @@ rec { }; }; - # 15. Create another layered image, for comparing layers with image 10. + # 16. Create another layered image, for comparing layers with image 10. another-layered-image = pkgs.dockerTools.buildLayeredImage { name = "another-layered-image"; tag = "latest"; config.Cmd = [ "${pkgs.hello}/bin/hello" ]; }; - # 16. Create a layered image with only 2 layers + # 17. Create a layered image with only 2 layers two-layered-image = pkgs.dockerTools.buildLayeredImage { name = "two-layered-image"; tag = "latest"; @@ -281,7 +309,7 @@ rec { maxLayers = 2; }; - # 17. Create a layered image with more packages than max layers. + # 18. Create a layered image with more packages than max layers. # coreutils and hello are part of the same layer bulk-layer = pkgs.dockerTools.buildLayeredImage { name = "bulk-layer"; @@ -292,27 +320,28 @@ rec { maxLayers = 2; }; - # 18. Create a "layered" image without nix store layers. This is not + # 19. Create a layered image with a base image and more packages than max + # layers. coreutils and hello are part of the same layer + layered-bulk-layer = pkgs.dockerTools.buildLayeredImage { + name = "layered-bulk-layer"; + tag = "latest"; + fromImage = two-layered-image; + contents = with pkgs; [ + coreutils hello + ]; + maxLayers = 4; + }; + + # 20. Create a "layered" image without nix store layers. This is not # recommended, but can be useful for base images in rare cases. no-store-paths = pkgs.dockerTools.buildLayeredImage { name = "no-store-paths"; tag = "latest"; extraCommands = '' - chmod a+w bin - # This removes sharing of busybox and is not recommended. We do this # to make the example suitable as a test case with working binaries. cp -r ${pkgs.pkgsStatic.busybox}/* . ''; - contents = [ - # This layer has no dependencies and its symlinks will be dereferenced - # when creating the customization layer. - (pkgs.runCommand "layer-to-flatten" {} '' - mkdir -p $out/bin - ln -s /bin/true $out/bin/custom-true - '' - ) - ]; }; nixLayered = pkgs.dockerTools.buildLayeredImageWithNixDb { @@ -335,7 +364,7 @@ rec { }; }; - # 19. Support files in the store on buildLayeredImage + # 21. Support files in the store on buildLayeredImage # See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223 filesInStore = pkgs.dockerTools.buildLayeredImageWithNixDb { name = "file-in-store"; @@ -355,7 +384,7 @@ rec { }; }; - # 20. Ensure that setting created to now results in a date which + # 22. Ensure that setting created to now results in a date which # isn't the epoch + 1 for layered images. unstableDateLayered = pkgs.dockerTools.buildLayeredImage { name = "unstable-date-layered"; @@ -415,7 +444,101 @@ rec { pkgs.dockerTools.buildLayeredImage { name = "bash-layered-with-user"; tag = "latest"; - contents = [ pkgs.bash pkgs.coreutils (nonRootShadowSetup { uid = 999; user = "somebody"; }) ]; + contents = [ pkgs.bash pkgs.coreutils ] ++ nonRootShadowSetup { uid = 999; user = "somebody"; }; }; + # basic example, with cross compilation + cross = let + # Cross compile for x86_64 if on aarch64 + crossPkgs = + if pkgs.system == "aarch64-linux" then pkgsCross.gnu64 + else pkgsCross.aarch64-multiplatform; + in crossPkgs.dockerTools.buildImage { + name = "hello-cross"; + tag = "latest"; + contents = crossPkgs.hello; + }; + + # layered image where a store path is itself a symlink + layeredStoreSymlink = + let + target = pkgs.writeTextDir "dir/target" "Content doesn't matter."; + symlink = pkgs.runCommandNoCC "symlink" {} "ln -s ${target} $out"; + in + pkgs.dockerTools.buildLayeredImage { + name = "layeredstoresymlink"; + tag = "latest"; + contents = [ pkgs.bash symlink ]; + } // { passthru = { inherit symlink; }; }; + + # image with registry/ prefix + prefixedImage = pkgs.dockerTools.buildImage { + name = "registry-1.docker.io/image"; + tag = "latest"; + config.Cmd = [ "${pkgs.hello}/bin/hello" ]; + }; + + # layered image with registry/ prefix + prefixedLayeredImage = pkgs.dockerTools.buildLayeredImage { + name = "registry-1.docker.io/layered-image"; + tag = "latest"; + config.Cmd = [ "${pkgs.hello}/bin/hello" ]; + }; + + # layered image with files owned by a user other than root + layeredImageWithFakeRootCommands = pkgs.dockerTools.buildLayeredImage { + name = "layered-image-with-fake-root-commands"; + tag = "latest"; + contents = [ + pkgs.pkgsStatic.busybox + ]; + fakeRootCommands = '' + mkdir -p ./home/jane + chown 1000 ./home/jane + ''; + }; + + # tarball consisting of both bash and redis images + mergedBashAndRedis = pkgs.dockerTools.mergeImages [ + bash + redis + ]; + + # tarball consisting of bash (without tag) and redis images + mergedBashNoTagAndRedis = pkgs.dockerTools.mergeImages [ + bashNoTag + redis + ]; + + # tarball consisting of bash and layered image with different owner of the + # /home/jane directory + mergedBashFakeRoot = pkgs.dockerTools.mergeImages [ + bash + layeredImageWithFakeRootCommands + ]; + + helloOnRoot = pkgs.dockerTools.streamLayeredImage { + name = "hello"; + tag = "latest"; + contents = [ + (pkgs.buildEnv { + name = "hello-root"; + paths = [ pkgs.hello ]; + }) + ]; + config.Cmd = [ "hello" ]; + }; + + helloOnRootNoStore = pkgs.dockerTools.streamLayeredImage { + name = "hello"; + tag = "latest"; + contents = [ + (pkgs.buildEnv { + name = "hello-root"; + paths = [ pkgs.hello ]; + }) + ]; + config.Cmd = [ "hello" ]; + includeStorePaths = false; + }; } diff --git a/pkgs/build-support/docker/nix-prefetch-docker b/pkgs/build-support/docker/nix-prefetch-docker index 1b6785189c2..5798ab5984f 100755 --- a/pkgs/build-support/docker/nix-prefetch-docker +++ b/pkgs/build-support/docker/nix-prefetch-docker @@ -127,7 +127,7 @@ trap "rm -rf \"$tmpPath\"" EXIT tmpFile="$tmpPath/$(get_name $finalImageName $finalImageTag)" if test -z "$QUIET"; then - skopeo --insecure-policy --tmpdir=$TMPDIR --override-os ${os} --override-arch ${arch} copy "$sourceUrl" "docker-archive://$tmpFile:$finalImageName:$finalImageTag" + skopeo --insecure-policy --tmpdir=$TMPDIR --override-os ${os} --override-arch ${arch} copy "$sourceUrl" "docker-archive://$tmpFile:$finalImageName:$finalImageTag" >&2 else skopeo --insecure-policy --tmpdir=$TMPDIR --override-os ${os} --override-arch ${arch} copy "$sourceUrl" "docker-archive://$tmpFile:$finalImageName:$finalImageTag" > /dev/null fi @@ -139,12 +139,12 @@ imageHash=$(nix-hash --flat --type $hashType --base32 "$tmpFile") finalPath=$(nix-store --add-fixed "$hashType" "$tmpFile") if test -z "$QUIET"; then - echo "-> ImageName: $imageName" - echo "-> ImageDigest: $imageDigest" - echo "-> FinalImageName: $finalImageName" - echo "-> FinalImageTag: $finalImageTag" - echo "-> ImagePath: $finalPath" - echo "-> ImageHash: $imageHash" + echo "-> ImageName: $imageName" >&2 + echo "-> ImageDigest: $imageDigest" >&2 + echo "-> FinalImageName: $finalImageName" >&2 + echo "-> FinalImageTag: $finalImageTag" >&2 + echo "-> ImagePath: $finalPath" >&2 + echo "-> ImageHash: $imageHash" >&2 fi if [ "$format" == "nix" ]; then diff --git a/pkgs/build-support/docker/nix-prefetch-docker.nix b/pkgs/build-support/docker/nix-prefetch-docker.nix index 6341eb0154b..61e917461ed 100644 --- a/pkgs/build-support/docker/nix-prefetch-docker.nix +++ b/pkgs/build-support/docker/nix-prefetch-docker.nix @@ -1,6 +1,4 @@ -{ stdenv, makeWrapper, nix, skopeo, jq }: - -with stdenv.lib; +{ lib, stdenv, makeWrapper, nix, skopeo, jq }: stdenv.mkDerivation { name = "nix-prefetch-docker"; @@ -12,13 +10,13 @@ stdenv.mkDerivation { installPhase = '' install -vD ${./nix-prefetch-docker} $out/bin/$name; wrapProgram $out/bin/$name \ - --prefix PATH : ${makeBinPath [ nix skopeo jq ]} \ + --prefix PATH : ${lib.makeBinPath [ nix skopeo jq ]} \ --set HOME /homeless-shelter ''; preferLocalBuild = true; - meta = { + meta = with lib; { description = "Script used to obtain source hashes for dockerTools.pullImage"; maintainers = with maintainers; [ offline ]; platforms = platforms.unix; diff --git a/pkgs/build-support/docker/stream_layered_image.py b/pkgs/build-support/docker/stream_layered_image.py index ffb6ba0ade4..d7c63eb43a7 100644 --- a/pkgs/build-support/docker/stream_layered_image.py +++ b/pkgs/build-support/docker/stream_layered_image.py @@ -45,21 +45,14 @@ from datetime import datetime, timezone from collections import namedtuple -def archive_paths_to(obj, paths, mtime, add_nix, filter=None): +def archive_paths_to(obj, paths, mtime): """ Writes the given store paths as a tar file to the given stream. obj: Stream to write to. Should have a 'write' method. paths: List of store paths. - add_nix: Whether /nix and /nix/store directories should be - prepended to the archive. - filter: An optional transformation to be applied to TarInfo - objects. Should take a single TarInfo object and return - another one. Defaults to identity. """ - filter = filter if filter else lambda i: i - # gettarinfo makes the paths relative, this makes them # absolute again def append_root(ti): @@ -72,7 +65,7 @@ def archive_paths_to(obj, paths, mtime, add_nix, filter=None): ti.gid = 0 ti.uname = "root" ti.gname = "root" - return filter(ti) + return ti def nix_root(ti): ti.mode = 0o0555 # r-xr-xr-x @@ -85,15 +78,17 @@ def archive_paths_to(obj, paths, mtime, add_nix, filter=None): with tarfile.open(fileobj=obj, mode="w|") as tar: # To be consistent with the docker utilities, we need to have - # these directories first when building layer tarballs. But - # we don't need them on the customisation layer. - if add_nix: - tar.addfile(apply_filters(nix_root(dir("/nix")))) - tar.addfile(apply_filters(nix_root(dir("/nix/store")))) + # these directories first when building layer tarballs. + tar.addfile(apply_filters(nix_root(dir("/nix")))) + tar.addfile(apply_filters(nix_root(dir("/nix/store")))) for path in paths: path = pathlib.Path(path) - files = itertools.chain([path], path.rglob("*")) + if path.is_symlink(): + files = [path] + else: + files = itertools.chain([path], path.rglob("*")) + for filename in sorted(files): ti = append_root(tar.gettarinfo(filename)) @@ -132,31 +127,104 @@ class ExtractChecksum: return (self._digest.hexdigest(), self._size) +FromImage = namedtuple("FromImage", ["tar", "manifest_json", "image_json"]) # Some metadata for a layer LayerInfo = namedtuple("LayerInfo", ["size", "checksum", "path", "paths"]) -def add_layer_dir(tar, paths, mtime, add_nix=True, filter=None): +def load_from_image(from_image_str): + """ + Loads the given base image, if any. + + from_image_str: Path to the base image archive. + + Returns: A 'FromImage' object with references to the loaded base image, + or 'None' if no base image was provided. + """ + if from_image_str is None: + return None + + base_tar = tarfile.open(from_image_str) + + manifest_json_tarinfo = base_tar.getmember("manifest.json") + with base_tar.extractfile(manifest_json_tarinfo) as f: + manifest_json = json.load(f) + + image_json_tarinfo = base_tar.getmember(manifest_json[0]["Config"]) + with base_tar.extractfile(image_json_tarinfo) as f: + image_json = json.load(f) + + return FromImage(base_tar, manifest_json, image_json) + + +def add_base_layers(tar, from_image): + """ + Adds the layers from the given base image to the final image. + + tar: 'tarfile.TarFile' object for new layers to be added to. + from_image: 'FromImage' object with references to the loaded base image. + """ + if from_image is None: + print("No 'fromImage' provided", file=sys.stderr) + return [] + + layers = from_image.manifest_json[0]["Layers"] + checksums = from_image.image_json["rootfs"]["diff_ids"] + layers_checksums = zip(layers, checksums) + + for num, (layer, checksum) in enumerate(layers_checksums, start=1): + layer_tarinfo = from_image.tar.getmember(layer) + checksum = re.sub(r"^sha256:", "", checksum) + + tar.addfile(layer_tarinfo, from_image.tar.extractfile(layer_tarinfo)) + path = layer_tarinfo.path + size = layer_tarinfo.size + + print("Adding base layer", num, "from", path, file=sys.stderr) + yield LayerInfo(size=size, checksum=checksum, path=path, paths=[path]) + + from_image.tar.close() + + +def overlay_base_config(from_image, final_config): + """ + Overlays the final image 'config' JSON on top of selected defaults from the + base image 'config' JSON. + + from_image: 'FromImage' object with references to the loaded base image. + final_config: 'dict' object of the final image 'config' JSON. + """ + if from_image is None: + return final_config + + base_config = from_image.image_json["config"] + + # Preserve environment from base image + final_env = base_config.get("Env", []) + final_config.get("Env", []) + if final_env: + # Resolve duplicates (last one wins) and format back as list + resolved_env = {entry.split("=", 1)[0]: entry for entry in final_env} + final_config["Env"] = list(resolved_env.values()) + return final_config + + +def add_layer_dir(tar, paths, store_dir, mtime): """ Appends given store paths to a TarFile object as a new layer. tar: 'tarfile.TarFile' object for the new layer to be added to. paths: List of store paths. + store_dir: the root directory of the nix store mtime: 'mtime' of the added files and the layer tarball. Should be an integer representing a POSIX time. - add_nix: Whether /nix and /nix/store directories should be - added to a layer. - filter: An optional transformation to be applied to TarInfo - objects inside the layer. Should take a single TarInfo - object and return another one. Defaults to identity. Returns: A 'LayerInfo' object containing some metadata of the layer added. """ - invalid_paths = [i for i in paths if not i.startswith("/nix/store/")] + invalid_paths = [i for i in paths if not i.startswith(store_dir)] assert len(invalid_paths) == 0, \ - "Expecting absolute store paths, but got: {invalid_paths}" + f"Expecting absolute paths from {store_dir}, but got: {invalid_paths}" # First, calculate the tarball checksum and the size. extract_checksum = ExtractChecksum() @@ -164,8 +232,6 @@ def add_layer_dir(tar, paths, mtime, add_nix=True, filter=None): extract_checksum, paths, mtime=mtime, - add_nix=add_nix, - filter=filter ) (checksum, size) = extract_checksum.extract() @@ -182,8 +248,6 @@ def add_layer_dir(tar, paths, mtime, add_nix=True, filter=None): write, paths, mtime=mtime, - add_nix=add_nix, - filter=filter ) write.close() @@ -199,29 +263,38 @@ def add_layer_dir(tar, paths, mtime, add_nix=True, filter=None): return LayerInfo(size=size, checksum=checksum, path=path, paths=paths) -def add_customisation_layer(tar, path, mtime): +def add_customisation_layer(target_tar, customisation_layer, mtime): """ - Adds the contents of the store path as a new layer. This is different - than the 'add_layer_dir' function defaults in the sense that the contents - of a single store path will be added to the root of the layer. eg (without - the /nix/store prefix). + Adds the customisation layer as a new layer. This is layer is structured + differently; given store path has the 'layer.tar' and corresponding + sha256sum ready. tar: 'tarfile.TarFile' object for the new layer to be added to. - path: A store path. - mtime: 'mtime' of the added files and the layer tarball. Should be an - integer representing a POSIX time. + customisation_layer: Path containing the layer archive. + mtime: 'mtime' of the added layer tarball. """ - def filter(ti): - ti.name = re.sub("^/nix/store/[^/]*", "", ti.name) - return ti - return add_layer_dir( - tar, - [path], - mtime=mtime, - add_nix=False, - filter=filter - ) + checksum_path = os.path.join(customisation_layer, "checksum") + with open(checksum_path) as f: + checksum = f.read().strip() + assert len(checksum) == 64, f"Invalid sha256 at ${checksum_path}." + + layer_path = os.path.join(customisation_layer, "layer.tar") + + path = f"{checksum}/layer.tar" + tarinfo = target_tar.gettarinfo(layer_path) + tarinfo.name = path + tarinfo.mtime = mtime + + with open(layer_path, "rb") as f: + target_tar.addfile(tarinfo, f) + + return LayerInfo( + size=None, + checksum=checksum, + path=path, + paths=[customisation_layer] + ) def add_bytes(tar, path, content, mtime): @@ -251,18 +324,23 @@ def main(): else datetime.fromisoformat(conf["created"]) ) mtime = int(created.timestamp()) + store_dir = conf["store_dir"] + + from_image = load_from_image(conf["from_image"]) with tarfile.open(mode="w|", fileobj=sys.stdout.buffer) as tar: layers = [] - for num, store_layer in enumerate(conf["store_layers"]): - print( - "Creating layer", num, - "from paths:", store_layer, - file=sys.stderr) - info = add_layer_dir(tar, store_layer, mtime=mtime) + layers.extend(add_base_layers(tar, from_image)) + + start = len(layers) + 1 + for num, store_layer in enumerate(conf["store_layers"], start=start): + print("Creating layer", num, "from paths:", store_layer, + file=sys.stderr) + info = add_layer_dir(tar, store_layer, store_dir, mtime=mtime) layers.append(info) - print("Creating the customisation layer...", file=sys.stderr) + print("Creating layer", len(layers) + 1, "with customisation...", + file=sys.stderr) layers.append( add_customisation_layer( tar, @@ -277,7 +355,7 @@ def main(): "created": datetime.isoformat(created), "architecture": conf["architecture"], "os": "linux", - "config": conf["config"], + "config": overlay_base_config(from_image, conf["config"]), "rootfs": { "diff_ids": [f"sha256:{layer.checksum}" for layer in layers], "type": "layers", diff --git a/pkgs/build-support/dotnetbuildhelpers/default.nix b/pkgs/build-support/dotnetbuildhelpers/default.nix index 809619ed55d..4348832ac04 100644 --- a/pkgs/build-support/dotnetbuildhelpers/default.nix +++ b/pkgs/build-support/dotnetbuildhelpers/default.nix @@ -1,4 +1,4 @@ -{ runCommand, mono, pkgconfig }: +{ runCommand, mono, pkg-config }: runCommand "dotnetbuildhelpers" { preferLocalBuild = true; } @@ -12,7 +12,7 @@ cp -v "$script" "$target"/"$scriptName" chmod 755 "$target"/"$scriptName" patchShebangs "$target"/"$scriptName" - substituteInPlace "$target"/"$scriptName" --replace pkg-config ${pkgconfig}/bin/${pkgconfig.targetPrefix}pkg-config + substituteInPlace "$target"/"$scriptName" --replace pkg-config ${pkg-config}/bin/${pkg-config.targetPrefix}pkg-config substituteInPlace "$target"/"$scriptName" --replace monodis ${mono}/bin/monodis done '' diff --git a/pkgs/build-support/dotnetenv/build-solution.nix b/pkgs/build-support/dotnetenv/build-solution.nix index 62370d361cd..b3372b94217 100644 --- a/pkgs/build-support/dotnetenv/build-solution.nix +++ b/pkgs/build-support/dotnetenv/build-solution.nix @@ -1,4 +1,4 @@ -{stdenv, dotnetfx}: +{ lib, stdenv, dotnetfx }: { name , src , baseDir ? "." @@ -16,20 +16,20 @@ assert modifyPublicMain -> mainClassFile != null; stdenv.mkDerivation { inherit name src; - - buildInputs = [ dotnetfx ]; + + buildInputs = [ dotnetfx ]; preConfigure = '' cd ${baseDir} ''; - + preBuild = '' - ${stdenv.lib.optionalString modifyPublicMain '' + ${lib.optionalString modifyPublicMain '' sed -i -e "s|static void Main|public static void Main|" ${mainClassFile} ''} ${preBuild} ''; - + installPhase = '' addDeps() { @@ -39,44 +39,44 @@ stdenv.mkDerivation { do windowsPath=$(cygpath --windows $i) assemblySearchPaths="$assemblySearchPaths;$windowsPath" - + addDeps $i done fi } - + for i in ${toString assemblyInputs} do - windowsPath=$(cygpath --windows $i) + windowsPath=$(cygpath --windows $i) echo "Using assembly path: $windowsPath" - + if [ "$assemblySearchPaths" = "" ] then assemblySearchPaths="$windowsPath" else assemblySearchPaths="$assemblySearchPaths;$windowsPath" fi - + addDeps $i done - + echo "Assembly search paths are: $assemblySearchPaths" - + if [ "$assemblySearchPaths" != "" ] then echo "Using assembly search paths args: $assemblySearchPathsArg" export AssemblySearchPaths=$assemblySearchPaths fi - + mkdir -p $out MSBuild.exe ${toString slnFile} /nologo /t:${targets} /p:IntermediateOutputPath=$(cygpath --windows $out)\\ /p:OutputPath=$(cygpath --windows $out)\\ /verbosity:${verbosity} ${options} - + # Because .NET assemblies store strings as UTF-16 internally, we cannot detect # hashes. Therefore a text files containing the proper paths is created # We can also use this file the propagate transitive dependencies. - + mkdir -p $out/nix-support - + for i in ${toString assemblyInputs} do echo $i >> $out/nix-support/dotnet-assemblies diff --git a/pkgs/build-support/dotnetenv/default.nix b/pkgs/build-support/dotnetenv/default.nix index 781a5ba8c0e..3015db42b07 100644 --- a/pkgs/build-support/dotnetenv/default.nix +++ b/pkgs/build-support/dotnetenv/default.nix @@ -1,16 +1,16 @@ -{stdenv, dotnetfx}: +{ lib, stdenv, dotnetfx }: let dotnetenv = { buildSolution = import ./build-solution.nix { - inherit stdenv; + inherit lib stdenv; dotnetfx = dotnetfx.pkg; }; buildWrapper = import ./wrapper.nix { inherit dotnetenv; }; - + inherit (dotnetfx) assembly20Path wcfPath referenceAssembly30Path referenceAssembly35Path; }; in diff --git a/pkgs/build-support/dotnetenv/wrapper.nix b/pkgs/build-support/dotnetenv/wrapper.nix index 4b07fc27dcb..423303c3084 100644 --- a/pkgs/build-support/dotnetenv/wrapper.nix +++ b/pkgs/build-support/dotnetenv/wrapper.nix @@ -36,25 +36,25 @@ dotnetenv.buildSolution { do windowsPath=$(cygpath --windows $i | sed 's|\\|\\\\|g') assemblySearchArray="$assemblySearchArray @\"$windowsPath\"" - + addRuntimeDeps $i done fi } - + export exePath=$(cygpath --windows $(find ${application} -name \*.exe) | sed 's|\\|\\\\|g') - + # Generate assemblySearchPaths string array contents for path in ${toString assemblyInputs} do assemblySearchArray="$assemblySearchArray @\"$(cygpath --windows $path | sed 's|\\|\\\\|g')\", " addRuntimeDeps $path done - + sed -e "s|@ROOTNAMESPACE@|${namespace}Wrapper|" \ -e "s|@ASSEMBLYNAME@|${namespace}|" \ Wrapper/Wrapper.csproj.in > Wrapper/Wrapper.csproj - + sed -e "s|@NAMESPACE@|${namespace}|g" \ -e "s|@MAINCLASSNAME@|${mainClassName}|g" \ -e "s|@EXEPATH@|$exePath|g" \ diff --git a/pkgs/build-support/emacs/elpa.nix b/pkgs/build-support/emacs/elpa.nix index e6f6c23e449..08257ff2542 100644 --- a/pkgs/build-support/emacs/elpa.nix +++ b/pkgs/build-support/emacs/elpa.nix @@ -1,16 +1,25 @@ # builder for Emacs packages built for packages.el -{ lib, stdenv, emacs, texinfo }: +{ lib, stdenv, emacs, texinfo, writeText, gcc }: with lib; { pname , version , src +, meta ? {} , ... }@args: -import ./generic.nix { inherit lib stdenv emacs texinfo; } ({ +let + + defaultMeta = { + homepage = args.src.meta.homepage or "https://elpa.gnu.org/packages/${pname}.html"; + }; + +in + +import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ phases = "installPhase fixupPhase distPhase"; @@ -19,10 +28,12 @@ import ./generic.nix { inherit lib stdenv emacs texinfo; } ({ emacs --batch -Q -l ${./elpa2nix.el} \ -f elpa2nix-install-package \ - "${src}" "$out/share/emacs/site-lisp/elpa" + "$src" "$out/share/emacs/site-lisp/elpa" runHook postInstall ''; + + meta = defaultMeta // meta; } // removeAttrs args [ "files" "fileSpecs" diff --git a/pkgs/build-support/emacs/setup-hook.sh b/pkgs/build-support/emacs/emacs-funcs.sh index 83e995631b3..e1e6a3b6220 100644 --- a/pkgs/build-support/emacs/setup-hook.sh +++ b/pkgs/build-support/emacs/emacs-funcs.sh @@ -7,9 +7,20 @@ addToEmacsLoadPath() { fi } +addToEmacsNativeLoadPath() { + local nativeDir="$1" + if [[ -d $nativeDir && ${EMACSNATIVELOADPATH-} != *"$nativeDir":* ]]; then + export EMACSNATIVELOADPATH="$nativeDir:${EMACSNATIVELOADPATH-}" + fi +} + addEmacsVars () { addToEmacsLoadPath "$1/share/emacs/site-lisp" + if [ -n "${addEmacsNativeLoadPath:-}" ]; then + addToEmacsNativeLoadPath "$1/share/emacs/native-lisp" + fi + # Add sub paths to the Emacs load path if it is a directory # containing .el files. This is necessary to build some packages, # e.g., using trivialBuild. @@ -21,13 +32,3 @@ addEmacsVars () { fi done } - -if [[ ! -v emacsHookDone ]]; then - emacsHookDone=1 - - # If this is for a wrapper derivation, emacs and the dependencies are all - # run-time dependencies. If this is for precompiling packages into bytecode, - # emacs is a compile-time dependency of the package. - addEnvHooks "$hostOffset" addEmacsVars - addEnvHooks "$targetOffset" addEmacsVars -fi diff --git a/pkgs/build-support/emacs/generic.nix b/pkgs/build-support/emacs/generic.nix index 956787ad59e..ef154982ad0 100644 --- a/pkgs/build-support/emacs/generic.nix +++ b/pkgs/build-support/emacs/generic.nix @@ -1,6 +1,6 @@ # generic builder for Emacs packages -{ lib, stdenv, emacs, texinfo }: +{ lib, stdenv, emacs, texinfo, writeText, gcc, ... }: with lib; @@ -49,7 +49,19 @@ stdenv.mkDerivation ({ propagatedBuildInputs = packageRequires; propagatedUserEnvPkgs = packageRequires; - setupHook = ./setup-hook.sh; + setupHook = writeText "setup-hook.sh" '' + source ${./emacs-funcs.sh} + + if [[ ! -v emacsHookDone ]]; then + emacsHookDone=1 + + # If this is for a wrapper derivation, emacs and the dependencies are all + # run-time dependencies. If this is for precompiling packages into bytecode, + # emacs is a compile-time dependency of the package. + addEnvHooks "$hostOffset" addEmacsVars + addEnvHooks "$targetOffset" addEmacsVars + fi + ''; doCheck = false; @@ -60,10 +72,22 @@ stdenv.mkDerivation ({ LIBRARY_PATH = "${lib.getLib stdenv.cc.libc}/lib"; + nativeBuildInputs = [ gcc ]; + + addEmacsNativeLoadPath = true; + postInstall = '' - find $out/share/emacs -type f -name '*.el' -print0 | xargs -0 -n 1 -I {} -P $NIX_BUILD_CORES sh -c "emacs --batch -f batch-native-compile {} || true" + # Besides adding the output directory to the native load path, make sure + # the current package's elisp files are in the load path, otherwise + # (require 'file-b) from file-a.el in the same package will fail. + mkdir -p $out/share/emacs/native-lisp + source ${./emacs-funcs.sh} + addEmacsVars "$out" + + find $out/share/emacs -type f -name '*.el' -print0 \ + | xargs -0 -n 1 -I {} -P $NIX_BUILD_CORES sh -c \ + "emacs --batch -f batch-native-compile {} || true" ''; - } // removeAttrs args [ "buildInputs" "packageRequires" diff --git a/pkgs/build-support/emacs/melpa.nix b/pkgs/build-support/emacs/melpa.nix index e2ec84c75e6..408448f26a0 100644 --- a/pkgs/build-support/emacs/melpa.nix +++ b/pkgs/build-support/emacs/melpa.nix @@ -1,7 +1,7 @@ # builder for Emacs packages built for packages.el # using MELPA package-build.el -{ lib, stdenv, fetchFromGitHub, emacs, texinfo }: +{ lib, stdenv, fetchFromGitHub, emacs, texinfo, writeText, gcc }: with lib; @@ -23,12 +23,12 @@ with lib; let defaultMeta = { - homepage = args.src.meta.homepage or "http://melpa.org/#/${pname}"; + homepage = args.src.meta.homepage or "https://melpa.org/#/${pname}"; }; in -import ./generic.nix { inherit lib stdenv emacs texinfo; } ({ +import ./generic.nix { inherit lib stdenv emacs texinfo writeText gcc; } ({ ename = if ename == null @@ -38,8 +38,8 @@ import ./generic.nix { inherit lib stdenv emacs texinfo; } ({ packageBuild = fetchFromGitHub { owner = "melpa"; repo = "package-build"; - rev = "0a22c3fbbf661822ec1791739953b937a12fa623"; - sha256 = "0dpy5p34il600sc8ic5jdgb3glya9si3lrvhxab0swks8fdydjgs"; + rev = "047801d301a73d4932f33f768d94a8ed26b8d524"; + sha256 = "0ygzkpg7xc3mjjbxg1kcyz6fwbkb0prvca499f0ffmhfaiv28h59"; }; elpa2nix = ./elpa2nix.el; @@ -70,7 +70,7 @@ import ./generic.nix { inherit lib stdenv emacs texinfo; } ({ -L "$NIX_BUILD_TOP/package-build" \ -l "$melpa2nix" \ -f melpa2nix-build-package \ - $ename $version + $ename $version $commit runHook postBuild ''; diff --git a/pkgs/build-support/emacs/melpa2nix.el b/pkgs/build-support/emacs/melpa2nix.el index 99c755e2afc..bd2aadd5aeb 100644 --- a/pkgs/build-support/emacs/melpa2nix.el +++ b/pkgs/build-support/emacs/melpa2nix.el @@ -12,5 +12,8 @@ (if (not noninteractive) (error "`melpa2nix-build-package' is to be used only with -batch")) (pcase command-line-args-left - (`(,package ,version) + (`(,package ,version ,commit) + ;; Monkey-patch package-build so it doesn't shell out to git/hg. + (defun package-build--get-commit (&rest _) + commit) (package-build--package (package-recipe-lookup package) version)))) diff --git a/pkgs/build-support/emacs/trivial.nix b/pkgs/build-support/emacs/trivial.nix index 396c971b2f0..f1aa078df27 100644 --- a/pkgs/build-support/emacs/trivial.nix +++ b/pkgs/build-support/emacs/trivial.nix @@ -1,12 +1,12 @@ # trivial builder for Emacs packages -{ lib, stdenv, texinfo, ... }@envargs: +{ callPackage, lib, ... }@envargs: with lib; args: -import ./generic.nix envargs ({ +callPackage ./generic.nix envargs ({ buildPhase = '' runHook preBuild diff --git a/pkgs/build-support/emacs/wrapper.nix b/pkgs/build-support/emacs/wrapper.nix index 1f2fbd8068e..6b53f3fdd95 100644 --- a/pkgs/build-support/emacs/wrapper.nix +++ b/pkgs/build-support/emacs/wrapper.nix @@ -2,11 +2,11 @@ # Usage -`emacsWithPackages` takes a single argument: a function from a package +`emacs.pkgs.withPackages` takes a single argument: a function from a package set to a list of packages (the packages that will be available in Emacs). For example, ``` -emacsWithPackages (epkgs: [ epkgs.evil epkgs.magit ]) +emacs.pkgs.withPackages (epkgs: [ epkgs.evil epkgs.magit ]) ``` All the packages in the list should come from the provided package set. It is possible to add any package to the list, but the provided @@ -15,26 +15,34 @@ the correct version of Emacs. # Overriding -`emacsWithPackages` inherits the package set which contains it, so the +`emacs.pkgs.withPackages` inherits the package set which contains it, so the correct way to override the provided package set is to override the -set which contains `emacsWithPackages`. For example, to override -`emacsPackages.emacsWithPackages`, +set which contains `emacs.pkgs.withPackages`. For example, to override +`emacs.pkgs.emacs.pkgs.withPackages`, ``` let customEmacsPackages = - emacsPackages.overrideScope' (self: super: { + emacs.pkgs.overrideScope' (self: super: { # use a custom version of emacs emacs = ...; # use the unstable MELPA version of magit magit = self.melpaPackages.magit; }); -in customEmacsPackages.emacsWithPackages (epkgs: [ epkgs.evil epkgs.magit ]) +in customEmacsPackages.emacs.pkgs.withPackages (epkgs: [ epkgs.evil epkgs.magit ]) ``` */ -{ lib, lndir, makeWrapper, runCommand }: self: +{ lib, lndir, makeWrapper, runCommand, gcc }: self: -with lib; let inherit (self) emacs; in +with lib; + +let + + inherit (self) emacs; + + nativeComp = emacs.nativeComp or false; + +in packagesFun: # packages explicitly requested by the user @@ -57,7 +65,10 @@ runCommand # Store all paths we want to add to emacs here, so that we only need to add # one path to the load lists deps = runCommand "emacs-packages-deps" - { inherit explicitRequires lndir emacs; } + { + inherit explicitRequires lndir emacs; + nativeBuildInputs = lib.optional nativeComp gcc; + } '' findInputsOld() { local pkg="$1"; shift @@ -95,6 +106,9 @@ runCommand } mkdir -p $out/bin mkdir -p $out/share/emacs/site-lisp + ${optionalString nativeComp '' + mkdir -p $out/share/emacs/native-lisp + ''} local requires for pkg in $explicitRequires; do @@ -116,6 +130,9 @@ runCommand linkEmacsPackage() { linkPath "$1" "bin" "bin" linkPath "$1" "share/emacs/site-lisp" "share/emacs/site-lisp" + ${optionalString nativeComp '' + linkPath "$1" "share/emacs/native-lisp" "share/emacs/native-lisp" + ''} } # Iterate over the array of inputs (avoiding nix's own interpolation) @@ -133,17 +150,32 @@ runCommand # Begin the new site-start.el by loading the original, which sets some # NixOS-specific paths. Paths are searched in the reverse of the order # they are specified in, so user and system profile paths are searched last. + # + # NOTE: Avoid displaying messages early at startup by binding + # inhibit-message to t. This would prevent the Emacs GUI from showing up + # prematurely. The messages would still be logged to the *Messages* + # buffer. rm -f $siteStart $siteStartByteCompiled $subdirs $subdirsByteCompiled cat >"$siteStart" <<EOF - (load-file "$emacs/share/emacs/site-lisp/site-start.el") + (let ((inhibit-message t)) + (load-file "$emacs/share/emacs/site-lisp/site-start.el")) (add-to-list 'load-path "$out/share/emacs/site-lisp") (add-to-list 'exec-path "$out/bin") + ${optionalString nativeComp '' + (add-to-list 'native-comp-eln-load-path "$out/share/emacs/native-lisp/") + ''} EOF # Link subdirs.el from the emacs distribution ln -s $emacs/share/emacs/site-lisp/subdirs.el -T $subdirs # Byte-compiling improves start-up time only slightly, but costs nothing. $emacs/bin/emacs --batch -f batch-byte-compile "$siteStart" "$subdirs" + + ${optionalString nativeComp '' + $emacs/bin/emacs --batch \ + --eval "(add-to-list 'native-comp-eln-load-path \"$out/share/emacs/native-lisp/\")" \ + -f batch-native-compile "$siteStart" "$subdirs" + ''} ''; inherit (emacs) meta; @@ -155,8 +187,13 @@ runCommand for prog in $emacs/bin/*; do # */ local progname=$(basename "$prog") rm -f "$out/bin/$progname" - makeWrapper "$prog" "$out/bin/$progname" \ - --suffix EMACSLOADPATH ":" "$deps/share/emacs/site-lisp:" + + substitute ${./wrapper.sh} $out/bin/$progname \ + --subst-var-by bash ${emacs.stdenv.shell} \ + --subst-var-by wrapperSiteLisp "$deps/share/emacs/site-lisp" \ + --subst-var-by wrapperSiteLispNative "$deps/share/emacs/native-lisp:" \ + --subst-var prog + chmod +x $out/bin/$progname done # Wrap MacOS app @@ -168,8 +205,13 @@ runCommand $emacs/Applications/Emacs.app/Contents/PkgInfo \ $emacs/Applications/Emacs.app/Contents/Resources \ $out/Applications/Emacs.app/Contents - makeWrapper $emacs/Applications/Emacs.app/Contents/MacOS/Emacs $out/Applications/Emacs.app/Contents/MacOS/Emacs \ - --suffix EMACSLOADPATH ":" "$deps/share/emacs/site-lisp:" + + + substitute ${./wrapper.sh} $out/Applications/Emacs.app/Contents/MacOS/Emacs \ + --subst-var-by bash ${emacs.stdenv.shell} \ + --subst-var-by wrapperSiteLisp "$deps/share/emacs/site-lisp" \ + --subst-var-by prog "$emacs/Applications/Emacs.app/Contents/MacOS/Emacs" + chmod +x $out/Applications/Emacs.app/Contents/MacOS/Emacs fi mkdir -p $out/share diff --git a/pkgs/build-support/emacs/wrapper.sh b/pkgs/build-support/emacs/wrapper.sh new file mode 100644 index 00000000000..e8eecb8c869 --- /dev/null +++ b/pkgs/build-support/emacs/wrapper.sh @@ -0,0 +1,47 @@ +#!@bash@ + +IFS=: + +newLoadPath=() +newNativeLoadPath=() +added= + +if [[ -n $EMACSLOADPATH ]] +then + while read -rd: entry + do + if [[ -z $entry && -z $added ]] + then + newLoadPath+=(@wrapperSiteLisp@) + added=1 + fi + newLoadPath+=("$entry") + done <<< "$EMACSLOADPATH:" +else + newLoadPath+=(@wrapperSiteLisp@) + newLoadPath+=("") +fi + +if [[ -n $EMACSNATIVELOADPATH ]] +then + while read -rd: entry + do + if [[ -z $entry && -z $added ]] + then + newNativeLoadPath+=(@wrapperSiteLispNative@) + added=1 + fi + newNativeLoadPath+=("$entry") + done <<< "$EMACSNATIVELOADPATH:" +else + newNativeLoadPath+=(@wrapperSiteLispNative@) + newNativeLoadPath+=("") +fi + +export EMACSLOADPATH="${newLoadPath[*]}" +export emacsWithPackages_siteLisp=@wrapperSiteLisp@ + +export EMACSNATIVELOADPATH="${newNativeLoadPath[*]}" +export emacsWithPackages_siteLispNative=@wrapperSiteLispNative@ + +exec @prog@ "$@" diff --git a/pkgs/build-support/expand-response-params/default.nix b/pkgs/build-support/expand-response-params/default.nix index 2a4bee74197..402f0071a53 100644 --- a/pkgs/build-support/expand-response-params/default.nix +++ b/pkgs/build-support/expand-response-params/default.nix @@ -10,7 +10,7 @@ stdenv.mkDerivation { src=$PWD ''; buildPhase = '' - "$CC" -std=c99 -O3 -o "expand-response-params" expand-response-params.c + NIX_CC_USE_RESPONSE_FILE=0 "$CC" -std=c99 -O3 -o "expand-response-params" expand-response-params.c ''; installPhase = '' mkdir -p $prefix/bin diff --git a/pkgs/build-support/fetchbitbucket/default.nix b/pkgs/build-support/fetchbitbucket/default.nix index a99f72e9eaa..e6e40c4379b 100644 --- a/pkgs/build-support/fetchbitbucket/default.nix +++ b/pkgs/build-support/fetchbitbucket/default.nix @@ -6,5 +6,4 @@ inherit name; url = "https://bitbucket.org/${owner}/${repo}/get/${rev}.tar.gz"; meta.homepage = "https://bitbucket.org/${owner}/${repo}/"; - extraPostFetch = ''rm -f "$out"/.hg_archival.txt''; # impure file; see #12002 } // removeAttrs args [ "owner" "repo" "rev" ]) // { inherit rev; } diff --git a/pkgs/build-support/fetchbzr/default.nix b/pkgs/build-support/fetchbzr/default.nix index 2cf169de7a5..b7db9e9274d 100644 --- a/pkgs/build-support/fetchbzr/default.nix +++ b/pkgs/build-support/fetchbzr/default.nix @@ -10,6 +10,6 @@ stdenvNoCC.mkDerivation { outputHashAlgo = "sha256"; outputHashMode = "recursive"; outputHash = sha256; - + inherit url rev; } diff --git a/pkgs/build-support/fetchcvs/builder.sh b/pkgs/build-support/fetchcvs/builder.sh index 0975c347882..fe1019aafc2 100644 --- a/pkgs/build-support/fetchcvs/builder.sh +++ b/pkgs/build-support/fetchcvs/builder.sh @@ -1,6 +1,6 @@ source $stdenv/setup -(echo '#!/usr/bin/env sh'; \ +(echo "#!$SHELL"; \ echo 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$@"') > ssh chmod +x ssh export CVS_RSH=$PWD/ssh diff --git a/pkgs/build-support/fetchfirefoxaddon/default.nix b/pkgs/build-support/fetchfirefoxaddon/default.nix new file mode 100644 index 00000000000..127f32dd61b --- /dev/null +++ b/pkgs/build-support/fetchfirefoxaddon/default.nix @@ -0,0 +1,41 @@ +{stdenv, lib, coreutils, unzip, jq, zip, fetchurl,writeScript, ...}: + +{ + name +, url +, md5 ? "" +, sha1 ? "" +, sha256 ? "" +, sha512 ? "" +, fixedExtid ? null +, hash ? "" +}: + +stdenv.mkDerivation rec { + + inherit name; + extid = if fixedExtid == null then "nixos@${name}" else fixedExtid; + passthru = { + inherit extid; + }; + + builder = writeScript "xpibuilder" '' + source $stdenv/setup + + header "firefox addon $name into $out" + + UUID="${extid}" + mkdir -p "$out/$UUID" + unzip -q ${src} -d "$out/$UUID" + NEW_MANIFEST=$(jq '. + {"applications": { "gecko": { "id": "${extid}" }}, "browser_specific_settings":{"gecko":{"id": "${extid}"}}}' "$out/$UUID/manifest.json") + echo "$NEW_MANIFEST" > "$out/$UUID/manifest.json" + cd "$out/$UUID" + zip -r -q -FS "$out/$UUID.xpi" * + rm -r "$out/$UUID" + ''; + src = fetchurl { + url = url; + inherit md5 sha1 sha256 sha512 hash; + }; + nativeBuildInputs = [ coreutils unzip zip jq ]; +} diff --git a/pkgs/build-support/fetchfossil/default.nix b/pkgs/build-support/fetchfossil/default.nix index 27933b47178..3a4876bc5de 100644 --- a/pkgs/build-support/fetchfossil/default.nix +++ b/pkgs/build-support/fetchfossil/default.nix @@ -1,11 +1,11 @@ -{stdenv, fossil}: +{stdenv, fossil, cacert}: {name ? null, url, rev, sha256}: stdenv.mkDerivation { name = "fossil-archive" + (if name != null then "-${name}" else ""); builder = ./builder.sh; - nativeBuildInputs = [fossil]; + nativeBuildInputs = [fossil cacert]; # Envvar docs are hard to find. A link for the future: # https://www.fossil-scm.org/index.html/doc/trunk/www/env-opts.md diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 6ae46469738..0047a335c76 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -8,6 +8,7 @@ header "exporting $url (rev $rev) into $out" $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" \ ${leaveDotGit:+--leave-dotGit} \ + ${fetchLFS:+--fetch-lfs} \ ${deepClone:+--deepClone} \ ${fetchSubmodules:+--fetch-submodules} \ ${branchName:+--branch-name "$branchName"} diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 0405951a9e4..3222866dc78 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -1,9 +1,9 @@ -{stdenvNoCC, git, cacert}: let +{lib, stdenvNoCC, git, git-lfs, cacert}: let urlToName = url: rev: let - inherit (stdenvNoCC.lib) removeSuffix splitString last; + inherit (lib) removeSuffix splitString last; base = last (splitString ":" (baseNameOf (removeSuffix "/" url))); - matched = builtins.match "(.*).git" base; + matched = builtins.match "(.*)\\.git" base; short = builtins.substring 0 7 rev; @@ -20,6 +20,7 @@ in # successfully. This can do things like check or transform the file. postFetch ? "" , preferLocalBuild ? true +, fetchLFS ? false }: /* NOTE: @@ -53,17 +54,19 @@ stdenvNoCC.mkDerivation { inherit name; builder = ./builder.sh; fetcher = ./nix-prefetch-git; # This must be a string to ensure it's called with bash. - nativeBuildInputs = [git]; + + nativeBuildInputs = [ git ] + ++ lib.optionals fetchLFS [ git-lfs ]; outputHashAlgo = "sha256"; outputHashMode = "recursive"; outputHash = sha256; - inherit url rev leaveDotGit fetchSubmodules deepClone branchName postFetch; + inherit url rev leaveDotGit fetchLFS fetchSubmodules deepClone branchName postFetch; GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars ++ [ + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "GIT_PROXY_COMMAND" "SOCKS_SERVER" ]; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index 43f7c5acd5a..8110d670e41 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -9,6 +9,7 @@ hashType=$NIX_HASH_ALGO deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT fetchSubmodules= +fetchLFS= builder= branchName=$NIX_PREFETCH_GIT_BRANCH_NAME @@ -46,6 +47,7 @@ Options: --deepClone Clone the entire repository. --no-deepClone Make a shallow clone of just the required ref. --leave-dotGit Keep the .git directories. + --fetch-lfs Fetch git Large File Storage (LFS) files. --fetch-submodules Fetch submodules. --builder Clone as fetchgit does, but url, rev, and out option are mandatory. --quiet Only print the final json summary. @@ -72,6 +74,7 @@ for arg; do --quiet) QUIET=true;; --no-deepClone) deepClone=;; --leave-dotGit) leaveDotGit=true;; + --fetch-lfs) fetchLFS=true;; --fetch-submodules) fetchSubmodules=true;; --builder) builder=true;; -h|--help) usage; exit;; @@ -103,7 +106,7 @@ fi init_remote(){ local url=$1 - clean_git init + clean_git init --initial-branch=master clean_git remote add origin "$url" ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true } @@ -137,7 +140,7 @@ url_to_name(){ fi } -# Fetch everything and checkout the right sha1 +# Fetch and checkout the right sha1 checkout_hash(){ local hash="$1" local ref="$2" @@ -146,8 +149,21 @@ checkout_hash(){ hash=$(hash_from_ref "$ref") fi + clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \ clean_git fetch -t ${builder:+--progress} origin || return 1 - clean_git checkout -b "$branchName" "$hash" || return 1 + + local object_type=$(git cat-file -t "$hash") + if [[ "$object_type" == "commit" ]]; then + clean_git checkout -b "$branchName" "$hash" || return 1 + elif [[ "$object_type" == "tree" ]]; then + clean_git config user.email "nix-prefetch-git@localhost" + clean_git config user.name "nix-prefetch-git" + local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash") + clean_git checkout -b "$branchName" "$commit_id" || return 1 + else + echo "Unrecognized git object type: $object_type" + return 1 + fi } # Fetch only a branch/tag and checkout it. @@ -283,6 +299,11 @@ clone_user_rev() { local url="$2" local rev="${3:-HEAD}" + if [ -n "$fetchLFS" ]; then + HOME=$TMPDIR + git lfs install + fi + # Perform the checkout. case "$rev" in HEAD|refs/*) diff --git a/pkgs/build-support/fetchgitea/default.nix b/pkgs/build-support/fetchgitea/default.nix new file mode 100644 index 00000000000..79804588cfe --- /dev/null +++ b/pkgs/build-support/fetchgitea/default.nix @@ -0,0 +1,7 @@ +# Gitea's URLs are compatible with GitHub + +{ lib, fetchFromGitHub }: + +{ domain, ... }@args: + +fetchFromGitHub ((removeAttrs args [ "domain" ]) // { githubBase = domain; }) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 66671dd0a6a..3f355d10f8a 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -1,17 +1,19 @@ { lib, fetchgit, fetchzip }: { owner, repo, rev, name ? "source" -, fetchSubmodules ? false, private ? false +, fetchSubmodules ? false, leaveDotGit ? null +, deepClone ? false, private ? false , githubBase ? "github.com", varPrefix ? null , ... # For hash agility -}@args: assert private -> !fetchSubmodules; +}@args: let baseUrl = "https://${githubBase}/${owner}/${repo}"; passthruAttrs = removeAttrs args [ "owner" "repo" "rev" "fetchSubmodules" "private" "githubBase" "varPrefix" ]; varBase = "NIX${if varPrefix == null then "" else "_${varPrefix}"}_GITHUB_PRIVATE_"; + useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone; # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. - fetcher = if fetchSubmodules then fetchgit else fetchzip; + fetcher = if useFetchGit then fetchgit else fetchzip; privateAttrs = lib.optionalAttrs private { netrcPhase = '' if [ -z "''$${varBase}USERNAME" -o -z "''$${varBase}PASSWORD" ]; then @@ -26,8 +28,14 @@ let ''; netrcImpureEnvVars = [ "${varBase}USERNAME" "${varBase}PASSWORD" ]; }; - fetcherArgs = (if fetchSubmodules - then { inherit rev fetchSubmodules; url = "${baseUrl}.git"; } + fetcherArgs = (if useFetchGit + then { + inherit rev deepClone fetchSubmodules; url = "${baseUrl}.git"; + } // lib.optionalAttrs (leaveDotGit != null) { inherit leaveDotGit; } else ({ url = "${baseUrl}/archive/${rev}.tar.gz"; } // privateAttrs) ) // passthruAttrs // { inherit name; }; -in fetcher fetcherArgs // { meta.homepage = baseUrl; inherit rev; } +in + +assert private -> !useFetchGit; + +fetcher fetcherArgs // { meta.homepage = baseUrl; inherit rev; } diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index f1850fbaa1e..77512510a7c 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -12,7 +12,7 @@ let ((optional (group != null) group) ++ [ owner repo ]); escapedSlug = replaceStrings ["." "/"] ["%2E" "%2F"] slug; - escapedRev = replaceStrings ["+"] ["%2B"] rev; + escapedRev = replaceStrings ["+" "%" "/"] ["%2B" "%25" "%2F"] rev; in fetchzip ({ diff --git a/pkgs/build-support/fetchhg/default.nix b/pkgs/build-support/fetchhg/default.nix index 41eff1f9c0c..15309d0a195 100644 --- a/pkgs/build-support/fetchhg/default.nix +++ b/pkgs/build-support/fetchhg/default.nix @@ -1,4 +1,4 @@ -{ stdenvNoCC, mercurial }: +{ lib, stdenvNoCC, mercurial }: { name ? null , url , rev ? null @@ -16,7 +16,7 @@ stdenvNoCC.mkDerivation { builder = ./builder.sh; nativeBuildInputs = [mercurial]; - impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; subrepoClause = if fetchSubrepos then "S" else ""; diff --git a/pkgs/build-support/fetchmavenartifact/default.nix b/pkgs/build-support/fetchmavenartifact/default.nix index 583a9ea396c..4274b4b52bf 100644 --- a/pkgs/build-support/fetchmavenartifact/default.nix +++ b/pkgs/build-support/fetchmavenartifact/default.nix @@ -1,12 +1,12 @@ # Adaptation of the MIT-licensed work on `sbt2nix` done by Charles O'Farrell -{ fetchurl, stdenv }: +{ lib, fetchurl, stdenv }: let defaultRepos = [ - "http://repo1.maven.org/maven2" - "http://oss.sonatype.org/content/repositories/releases" - "http://oss.sonatype.org/content/repositories/public" - "http://repo.typesafe.com/typesafe/releases" + "https://repo1.maven.org/maven2" + "https://oss.sonatype.org/content/repositories/releases" + "https://oss.sonatype.org/content/repositories/public" + "https://repo.typesafe.com/typesafe/releases" ]; in @@ -17,6 +17,8 @@ args@ artifactId , # Example: "4.3.6" version +, # Example: "jdk11" + classifier ? null , # List of maven repositories from where to fetch the artifact. # Example: [ http://oss.sonatype.org/content/repositories/public ]. repos ? defaultRepos @@ -34,21 +36,20 @@ assert (url == "") || (urls == []); # if repos is empty, then url or urls must be specified. assert (repos != []) || (url != "") || (urls != []); - let name_ = - with stdenv.lib; concatStrings [ - (replaceChars ["."] ["_"] groupId) "_" - (replaceChars ["."] ["_"] artifactId) "-" + lib.concatStrings [ + (lib.replaceChars ["."] ["_"] groupId) "_" + (lib.replaceChars ["."] ["_"] artifactId) "-" version ]; mkJarUrl = repoUrl: - with stdenv.lib; concatStringsSep "/" [ - (removeSuffix "/" repoUrl) - (replaceChars ["."] ["/"] groupId) + lib.concatStringsSep "/" [ + (lib.removeSuffix "/" repoUrl) + (lib.replaceChars ["."] ["/"] groupId) artifactId version - "${artifactId}-${version}.jar" + "${artifactId}-${version}${lib.optionalString (!isNull classifier) "-${classifier}"}.jar" ]; urls_ = if url != "" then [url] @@ -56,7 +57,7 @@ let else map mkJarUrl repos; jar = fetchurl ( - builtins.removeAttrs args ["groupId" "artifactId" "version" "repos" "url" ] + builtins.removeAttrs args ["groupId" "artifactId" "version" "classifier" "repos" "url" ] // { urls = urls_; name = "${name_}.jar"; } ); in diff --git a/pkgs/build-support/fetchmtn/default.nix b/pkgs/build-support/fetchmtn/default.nix index 7ce67453d69..4aa134242aa 100644 --- a/pkgs/build-support/fetchmtn/default.nix +++ b/pkgs/build-support/fetchmtn/default.nix @@ -1,10 +1,10 @@ # You can specify some extra mirrors and a cache DB via options -{stdenvNoCC, monotone, defaultDBMirrors ? [], cacheDB ? "./mtn-checkout.db"}: +{lib, stdenvNoCC, monotone, defaultDBMirrors ? [], cacheDB ? "./mtn-checkout.db"}: # dbs is a list of strings # each is an url for sync # selector is mtn selector, like h:org.example.branch -# +# {name ? "mtn-checkout", dbs ? [], sha256 , selector ? "h:" + branch, branch}: @@ -19,7 +19,7 @@ stdenvNoCC.mkDerivation { dbs = defaultDBMirrors ++ dbs; inherit branch cacheDB name selector; - impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; } diff --git a/pkgs/build-support/fetchnuget/default.nix b/pkgs/build-support/fetchnuget/default.nix index 960bc3c1ffb..ad61b9a51d2 100644 --- a/pkgs/build-support/fetchnuget/default.nix +++ b/pkgs/build-support/fetchnuget/default.nix @@ -19,7 +19,7 @@ else sourceRoot = "."; - buildInputs = [ unzip ]; + nativeBuildInputs = [ unzip ]; dontBuild = true; diff --git a/pkgs/build-support/fetchrepoproject/default.nix b/pkgs/build-support/fetchrepoproject/default.nix index 8144ed038bd..69b1bd1aef7 100644 --- a/pkgs/build-support/fetchrepoproject/default.nix +++ b/pkgs/build-support/fetchrepoproject/default.nix @@ -1,4 +1,4 @@ -{ stdenvNoCC, gitRepo, cacert, copyPathsToStore }: +{ lib, stdenvNoCC, gitRepo, cacert, copyPathsToStore }: { name, manifest, rev ? "HEAD", sha256 # Optional parameters: @@ -9,7 +9,7 @@ assert repoRepoRev != "" -> repoRepoURL != ""; assert createMirror -> !useArchive; -with stdenvNoCC.lib; +with lib; let extraRepoInitFlags = [ diff --git a/pkgs/build-support/fetchs3/default.nix b/pkgs/build-support/fetchs3/default.nix index 3dbde203374..acad0749b66 100644 --- a/pkgs/build-support/fetchs3/default.nix +++ b/pkgs/build-support/fetchs3/default.nix @@ -1,4 +1,4 @@ -{ stdenvNoCC, runCommand, awscli }: +{ lib, runCommand, awscli }: { s3url , name ? builtins.baseNameOf s3url @@ -16,7 +16,7 @@ let AWS_SESSION_TOKEN = session_token; }; - credentialAttrs = stdenvNoCC.lib.optionalAttrs (credentials != null) (mkCredentials credentials); + credentialAttrs = lib.optionalAttrs (credentials != null) (mkCredentials credentials); in runCommand name ({ nativeBuildInputs = [ awscli ]; diff --git a/pkgs/build-support/fetchsourcehut/default.nix b/pkgs/build-support/fetchsourcehut/default.nix new file mode 100644 index 00000000000..ed2f074200c --- /dev/null +++ b/pkgs/build-support/fetchsourcehut/default.nix @@ -0,0 +1,25 @@ +{ fetchzip, lib }: + +{ owner +, repo, rev +, domain ? "sr.ht" +, vc ? "git" +, name ? "source" +, ... # For hash agility +} @ args: + +with lib; + +assert (lib.assertOneOf "vc" vc [ "hg" "git" ]); + +let + baseUrl = "https://${vc}.${domain}/${owner}/${repo}"; + +in fetchzip (recursiveUpdate { + inherit name; + url = "${baseUrl}/archive/${rev}.tar.gz"; + meta.homepage = "${baseUrl}/"; + extraPostFetch = optionalString (vc == "hg") '' + rm -f "$out/.hg_archival.txt" + ''; # impure file; see #12002 +} (removeAttrs args [ "owner" "repo" "rev" "domain" "vc" ])) // { inherit rev; } diff --git a/pkgs/build-support/fetchsvn/default.nix b/pkgs/build-support/fetchsvn/default.nix index 06f0ea0a3d1..82dececc124 100644 --- a/pkgs/build-support/fetchsvn/default.nix +++ b/pkgs/build-support/fetchsvn/default.nix @@ -1,4 +1,4 @@ -{ stdenvNoCC, buildPackages +{ lib, stdenvNoCC, buildPackages , subversion, glibcLocales, sshSupport ? true, openssh ? null }: @@ -10,7 +10,7 @@ assert sshSupport -> openssh != null; let - repoName = with stdenvNoCC.lib; + repoName = with lib; let fst = head; snd = l: head (tail l); @@ -39,7 +39,7 @@ stdenvNoCC.mkDerivation { name = name_; builder = ./builder.sh; nativeBuildInputs = [ subversion glibcLocales ] - ++ stdenvNoCC.lib.optional sshSupport openssh; + ++ lib.optional sshSupport openssh; SVN_SSH = if sshSupport then "${buildPackages.openssh}/bin/ssh" else null; @@ -49,6 +49,6 @@ stdenvNoCC.mkDerivation { inherit url rev ignoreExternals ignoreKeywords; - impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; inherit preferLocalBuild; } diff --git a/pkgs/build-support/fetchsvnrevision/default.nix b/pkgs/build-support/fetchsvnrevision/default.nix index 288451a225c..f2e2a11da8d 100644 --- a/pkgs/build-support/fetchsvnrevision/default.nix +++ b/pkgs/build-support/fetchsvnrevision/default.nix @@ -7,4 +7,4 @@ runCommand: subversion: repository: rev=$(echo p | svn ls -v --depth empty ${repository} |awk '{ print $1 }') echo "[ \"$rev\" ]" > $out echo Latest revision is $rev - '') \ No newline at end of file + '') diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index c65738aef41..8ce69a7f187 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -104,7 +104,9 @@ let if urls != [] && url == "" then (if lib.isList urls then urls else throw "`urls` is not a list") - else if urls == [] && url != "" then [url] + else if urls == [] && url != "" then + (if lib.isString url then [url] + else throw "`url` is not a string") else throw "fetchurl requires either `url` or `urls` to be set"; hash_ = diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 0eba8816b63..e5095478002 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -9,19 +9,19 @@ # "sourceforge", "gnu", etc. luarocks = [ - "https://luarocks.org" + "https://luarocks.org/" "https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/" - "http://luafr.org/moonrocks" - "http://luarocks.logiceditor.com/rocks" + "http://luafr.org/moonrocks/" + "http://luarocks.logiceditor.com/rocks/" ]; # SourceForge. sourceforge = [ "https://downloads.sourceforge.net/" "https://prdownloads.sourceforge.net/" - "https://heanet.dl.sourceforge.net/sourceforge/" - "https://surfnet.dl.sourceforge.net/sourceforge/" - "https://dfn.dl.sourceforge.net/sourceforge/" + "https://netcologne.dl.sourceforge.net/sourceforge/" + "https://versaweb.dl.sourceforge.net/sourceforge/" + "https://freefr.dl.sourceforge.net/sourceforge/" "https://osdn.dl.sourceforge.net/sourceforge/" "https://kent.dl.sourceforge.net/sourceforge/" ]; @@ -417,6 +417,11 @@ "https://pypi.io/packages/source/" ]; + # Python Test-PyPI mirror + testpypi = [ + "https://test.pypi.io/packages/source/" + ]; + # Mozilla projects. mozilla = [ "http://download.cdn.mozilla.net/pub/mozilla.org/" diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index c61df8ceb00..cde4d4f579f 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -5,16 +5,19 @@ # (e.g. due to minor changes in the compression algorithm, or changes # in timestamps). -{ fetchurl, unzip }: +{ lib, fetchurl, unzip }: { # Optionally move the contents of the unpacked tree up one level. stripRoot ? true -, url +, url ? "" +, urls ? [] , extraPostFetch ? "" , name ? "source" , ... } @ args: -(fetchurl ({ +(fetchurl (let + basename = baseNameOf (if url != "" then url else builtins.head urls); +in { inherit name; recursiveHash = true; @@ -27,7 +30,7 @@ mkdir "$unpackDir" cd "$unpackDir" - renamed="$TMPDIR/${baseNameOf url}" + renamed="$TMPDIR/${basename}" mv "$downloadedFile" "$renamed" unpackFile "$renamed" '' @@ -44,8 +47,15 @@ mv "$unpackDir/$fn" "$out" '' else '' mv "$unpackDir" "$out" - '') #*/ - + extraPostFetch; + '') + + '' + ${extraPostFetch} + '' + # Remove non-owner write permissions + # Fixes https://github.com/NixOS/nixpkgs/issues/38649 + + '' + chmod 755 "$out" + ''; } // removeAttrs args [ "stripRoot" "extraPostFetch" ])).overrideAttrs (x: { # Hackety-hack: we actually need unzip hooks, too nativeBuildInputs = x.nativeBuildInputs ++ [ unzip ]; diff --git a/pkgs/build-support/go/garble.nix b/pkgs/build-support/go/garble.nix new file mode 100644 index 00000000000..da1e3152ba4 --- /dev/null +++ b/pkgs/build-support/go/garble.nix @@ -0,0 +1,34 @@ +{ stdenv +, buildGoModule +, fetchFromGitHub +, lib +}: +buildGoModule rec { + pname = "garble"; + version = "20200107"; + + src = fetchFromGitHub { + owner = "burrowers"; + repo = pname; + rev = "835f4aadf321521acf06aac4d5068473dc4b2ac1"; + sha256 = "sha256-NodsVHRll2YZoxrhmniJvelQOStG82u3kJyc0t8OXD8="; + }; + + vendorSha256 = "sha256-x2fk2QmZDK2yjyfYdK7x+sQjvt7tuggmm8ieVjsNKek="; + + preBuild = '' + # https://github.com/burrowers/garble/issues/184 + substituteInPlace testdata/scripts/tiny.txt \ + --replace "{6,8}" "{4,8}" + '' + lib.optionalString (!stdenv.isx86_64) '' + # The test assumex amd64 assembly + rm testdata/scripts/asm.txt + ''; + + meta = { + description = "Obfuscate Go code by wrapping the Go toolchain"; + homepage = "https://github.com/burrowers/garble/"; + maintainers = with lib.maintainers; [ davhau ]; + license = lib.licenses.bsd3; + }; +} diff --git a/pkgs/build-support/icon-conv-tools/default.nix b/pkgs/build-support/icon-conv-tools/default.nix index 0ea18d8768a..442f1f2235f 100644 --- a/pkgs/build-support/icon-conv-tools/default.nix +++ b/pkgs/build-support/icon-conv-tools/default.nix @@ -1,7 +1,8 @@ -{ stdenv, icoutils }: +{ lib, stdenv, icoutils }: stdenv.mkDerivation { - name = "icon-conv-tools-0.0.0"; + pname = "icon-conv-tools"; + version = "0.0.0"; src = ./bin; @@ -23,9 +24,9 @@ stdenv.mkDerivation { dontPatchELF = true; dontStrip = true; - meta = { + meta = with lib; { description = "Tools for icon conversion specific to nix package manager"; - maintainers = with stdenv.lib.maintainers; [ jraygauthier ]; - platforms = with stdenv.lib.platforms; linux; + maintainers = with maintainers; [ jraygauthier ]; + platforms = platforms.linux; }; } diff --git a/pkgs/build-support/install-shell-files/default.nix b/pkgs/build-support/install-shell-files/default.nix index e1f2e24dd87..d50661ddc65 100644 --- a/pkgs/build-support/install-shell-files/default.nix +++ b/pkgs/build-support/install-shell-files/default.nix @@ -1,4 +1,12 @@ -{ makeSetupHook }: +{ makeSetupHook, tests }: # See the header comment in ../setup-hooks/install-shell-files.sh for example usage. -makeSetupHook { name = "install-shell-files"; } ../setup-hooks/install-shell-files.sh +let + setupHook = makeSetupHook { name = "install-shell-files"; } ../setup-hooks/install-shell-files.sh; +in + +setupHook.overrideAttrs (oldAttrs: { + passthru = (oldAttrs.passthru or {}) // { + tests = tests.install-shell-files; + }; +}) diff --git a/pkgs/build-support/kernel/initrd-compressor-meta.nix b/pkgs/build-support/kernel/initrd-compressor-meta.nix new file mode 100644 index 00000000000..443e599a239 --- /dev/null +++ b/pkgs/build-support/kernel/initrd-compressor-meta.nix @@ -0,0 +1,53 @@ +rec { + cat = { + executable = pkgs: "cat"; + ubootName = "none"; + extension = ".cpio"; + }; + gzip = { + executable = pkgs: "${pkgs.gzip}/bin/gzip"; + defaultArgs = ["-9n"]; + ubootName = "gzip"; + extension = ".gz"; + }; + bzip2 = { + executable = pkgs: "${pkgs.bzip2}/bin/bzip2"; + ubootName = "bzip2"; + extension = ".bz2"; + }; + xz = { + executable = pkgs: "${pkgs.xz}/bin/xz"; + defaultArgs = ["--check=crc32" "--lzma2=dict=512KiB"]; + extension = ".xz"; + }; + lzma = { + executable = pkgs: "${pkgs.xz}/bin/lzma"; + defaultArgs = ["--check=crc32" "--lzma1=dict=512KiB"]; + ubootName = "lzma"; + extension = ".lzma"; + }; + lz4 = { + executable = pkgs: "${pkgs.lz4}/bin/lz4"; + defaultArgs = ["-l"]; + ubootName = "lz4"; + extension = ".lz4"; + }; + lzop = { + executable = pkgs: "${pkgs.lzop}/bin/lzop"; + ubootName = "lzo"; + extension = ".lzo"; + }; + zstd = { + executable = pkgs: "${pkgs.zstd}/bin/zstd"; + defaultArgs = ["-10"]; + ubootName = "zstd"; + extension = ".zst"; + }; + pigz = gzip // { + executable = pkgs: "${pkgs.pigz}/bin/pigz"; + }; + pixz = xz // { + executable = pkgs: "${pkgs.pixz}/bin/pixz"; + defaultArgs = []; + }; +} diff --git a/pkgs/build-support/kernel/make-initrd.nix b/pkgs/build-support/kernel/make-initrd.nix index ed5dbdaee17..83d3bb65bae 100644 --- a/pkgs/build-support/kernel/make-initrd.nix +++ b/pkgs/build-support/kernel/make-initrd.nix @@ -1,22 +1,74 @@ -# Create an initial ramdisk containing the closure of the specified -# file system objects. An initial ramdisk is used during the initial +# Create an initramfs containing the closure of the specified +# file system objects. An initramfs is used during the initial # stages of booting a Linux system. It is loaded by the boot loader # along with the kernel image. It's supposed to contain everything # (such as kernel modules) necessary to allow us to mount the root # file system. Once the root file system is mounted, the `real' boot # script can be called. # -# An initrd is really just a gzipped cpio archive. -# -# Symlinks are created for each top-level file system object. E.g., -# `contents = {object = ...; symlink = /init;}' is a typical -# argument. - -{ stdenvNoCC, perl, cpio, contents, ubootTools +# An initramfs is a cpio archive, and may be compressed with a number +# of algorithms. +let + # Some metadata on various compression programs, relevant to naming + # the initramfs file and, if applicable, generating a u-boot image + # from it. + compressors = import ./initrd-compressor-meta.nix; + # Get the basename of the actual compression program from the whole + # compression command, for the purpose of guessing the u-boot + # compression type and filename extension. + compressorName = fullCommand: builtins.elemAt (builtins.match "([^ ]*/)?([^ ]+).*" fullCommand) 1; +in +{ stdenvNoCC, perl, cpio, ubootTools, lib, pkgsBuildHost +# Name of the derivation (not of the resulting file!) , name ? "initrd" -, compressor ? "gzip -9n" + +# Program used to compress the cpio archive; use "cat" for no compression. +# This can also be a function which takes a package set and returns the path to the compressor, +# such as `pkgs: "${pkgs.lzop}/bin/lzop"`. +, compressor ? "gzip" +, _compressorFunction ? + if lib.isFunction compressor then compressor + else if ! builtins.hasContext compressor && builtins.hasAttr compressor compressors then compressors.${compressor}.executable + else _: compressor +, _compressorExecutable ? _compressorFunction pkgsBuildHost +, _compressorName ? compressorName _compressorExecutable +, _compressorMeta ? compressors.${_compressorName} or {} + +# List of arguments to pass to the compressor program, or null to use its defaults +, compressorArgs ? null +, _compressorArgsReal ? if compressorArgs == null then _compressorMeta.defaultArgs or [] else compressorArgs + +# Filename extension to use for the compressed initramfs. This is +# included for clarity, but $out/initrd will always be a symlink to +# the final image. +# If this isn't guessed, you may want to complete the metadata above and send a PR :) +, extension ? _compressorMeta.extension or + (throw "Unrecognised compressor ${_compressorName}, please specify filename extension") + +# List of { object = path_or_derivation; symlink = "/path"; } +# The paths are copied into the initramfs in their nix store path +# form, then linked at the root according to `symlink`. +, contents + +# List of uncompressed cpio files to prepend to the initramfs. This +# can be used to add files in specified paths without them becoming +# symlinks to store paths. , prepend ? [] -, lib + +# Whether to wrap the initramfs in a u-boot image. +, makeUInitrd ? stdenvNoCC.hostPlatform.linux-kernel.target == "uImage" + +# If generating a u-boot image, the architecture to use. The default +# guess may not align with u-boot's nomenclature correctly, so it can +# be overridden. +# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 for a list. +, uInitrdArch ? stdenvNoCC.hostPlatform.linuxArch + +# The name of the compression, as recognised by u-boot. +# See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L195-204 for a list. +# If this isn't guessed, you may want to complete the metadata above and send a PR :) +, uInitrdCompression ? _compressorMeta.ubootName or + (throw "Unrecognised compressor ${_compressorName}, please specify uInitrdCompression") }: let # !!! Move this into a public lib function, it is probably useful for others @@ -24,14 +76,25 @@ let lib.concatStringsSep "-" (filter (x: !(isList x)) (split "[^a-zA-Z0-9_=.?-]+" x)); in stdenvNoCC.mkDerivation rec { - inherit name; + inherit name makeUInitrd extension uInitrdArch prepend; - builder = ./make-initrd.sh; + ${if makeUInitrd then "uinitrdCompression" else null} = uInitrdCompression; - makeUInitrd = stdenvNoCC.hostPlatform.platform.kernelTarget == "uImage"; + builder = ./make-initrd.sh; nativeBuildInputs = [ perl cpio ] - ++ stdenvNoCC.lib.optional makeUInitrd ubootTools; + ++ lib.optional makeUInitrd ubootTools; + + compress = "${_compressorExecutable} ${lib.escapeShellArgs _compressorArgsReal}"; + + # Pass the function through, for reuse in append-initrd-secrets. The + # function is used instead of the string, in order to support + # cross-compilation (append-initrd-secrets running on a different + # architecture than what the main initramfs is built on). + passthru = { + compressorExecutableFunction = _compressorFunction; + compressorArgs = _compressorArgsReal; + }; # !!! should use XML. objects = map (x: x.object) contents; @@ -47,6 +110,4 @@ in stdenvNoCC.mkDerivation rec { contents (lib.range 0 (lib.length contents - 1)); pathsFromGraph = ./paths-from-graph.pl; - - inherit compressor prepend; } diff --git a/pkgs/build-support/kernel/make-initrd.sh b/pkgs/build-support/kernel/make-initrd.sh index 0aeaedeb372..c0619ef14ae 100644 --- a/pkgs/build-support/kernel/make-initrd.sh +++ b/pkgs/build-support/kernel/make-initrd.sh @@ -39,10 +39,13 @@ mkdir -p $out for PREP in $prepend; do cat $PREP >> $out/initrd done -(cd root && find * -print0 | xargs -0r touch -h -d '@1') -(cd root && find * -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | $compressor >> $out/initrd) +(cd root && find * .[^.*] -exec touch -h -d '@1' '{}' +) +(cd root && find * .[^.*] -print0 | sort -z | cpio -o -H newc -R +0:+0 --reproducible --null | eval -- $compress >> "$out/initrd") if [ -n "$makeUInitrd" ]; then - mv $out/initrd $out/initrd.gz - mkimage -A arm -O linux -T ramdisk -C gzip -d $out/initrd.gz $out/initrd + mkimage -A $uInitrdArch -O linux -T ramdisk -C "$uInitrdCompression" -d $out/initrd"$extension" $out/initrd.img + # Compatibility symlink + ln -s "initrd.img" "$out/initrd" +else + ln -s "initrd" "$out/initrd$extension" fi diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh index 68d840f1614..3b3a38ea1d3 100644 --- a/pkgs/build-support/kernel/modules-closure.sh +++ b/pkgs/build-support/kernel/modules-closure.sh @@ -68,10 +68,21 @@ done mkdir -p $out/lib/firmware for module in $(cat closure); do - for i in $(modinfo -F firmware $module); do + # for builtin modules, modinfo will reply with a wrong output looking like: + # $ modinfo -F firmware unix + # name: unix + # + # There is a pending attempt to fix this: + # https://github.com/NixOS/nixpkgs/pull/96153 + # https://lore.kernel.org/linux-modules/20200823215433.j5gc5rnsmahpf43v@blumerang/T/#u + # + # For now, the workaround is just to filter out the extraneous lines out + # of its output. + for i in $(modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:'); do mkdir -p "$out/lib/firmware/$(dirname "$i")" echo "firmware for $module: $i" - cp "$firmware/lib/firmware/$i" "$out/lib/firmware/$i" 2>/dev/null || if test -z "$allowMissing"; then exit 1; fi + cp "$firmware/lib/firmware/$i" "$out/lib/firmware/$i" 2>/dev/null \ + || echo "WARNING: missing firmware $i for module $module" done done diff --git a/pkgs/build-support/libredirect/default.nix b/pkgs/build-support/libredirect/default.nix index 6e54e2a696c..4678d35442f 100644 --- a/pkgs/build-support/libredirect/default.nix +++ b/pkgs/build-support/libredirect/default.nix @@ -1,7 +1,8 @@ { stdenv, lib, coreutils }: -stdenv.mkDerivation { - name = "libredirect-0"; +stdenv.mkDerivation rec { + pname = "libredirect"; + version = "0"; unpackPhase = '' cp ${./libredirect.c} libredirect.c @@ -46,8 +47,8 @@ stdenv.mkDerivation { ) ''; - meta = { - platforms = stdenv.lib.platforms.unix; + meta = with lib; { + platforms = platforms.unix; description = "An LD_PRELOAD library to intercept and rewrite the paths in glibc calls"; longDescription = '' libredirect is an LD_PRELOAD library to intercept and rewrite the paths in diff --git a/pkgs/build-support/libredirect/libredirect.c b/pkgs/build-support/libredirect/libredirect.c index e7f74c736ab..dfa2978e9f4 100644 --- a/pkgs/build-support/libredirect/libredirect.c +++ b/pkgs/build-support/libredirect/libredirect.c @@ -9,6 +9,7 @@ #include <limits.h> #include <string.h> #include <spawn.h> +#include <dirent.h> #define MAX_REDIRECTS 128 @@ -121,6 +122,13 @@ FILE * fopen(const char * path, const char * mode) return fopen_real(rewrite(path, buf), mode); } +FILE * __nss_files_fopen(const char * path) +{ + FILE * (*__nss_files_fopen_real) (const char *) = dlsym(RTLD_NEXT, "__nss_files_fopen"); + char buf[PATH_MAX]; + return __nss_files_fopen_real(rewrite(path, buf)); +} + FILE * fopen64(const char * path, const char * mode) { FILE * (*fopen64_real) (const char *, const char *) = dlsym(RTLD_NEXT, "fopen64"); @@ -182,9 +190,85 @@ int posix_spawnp(pid_t * pid, const char * file, return posix_spawnp_real(pid, rewrite(file, buf), file_actions, attrp, argv, envp); } -int execv(const char *path, char *const argv[]) +int execv(const char * path, char * const argv[]) { - int (*execv_real) (const char *path, char *const argv[]) = dlsym(RTLD_NEXT, "execv"); + int (*execv_real) (const char * path, char * const argv[]) = dlsym(RTLD_NEXT, "execv"); char buf[PATH_MAX]; return execv_real(rewrite(path, buf), argv); } + +int execvp(const char * path, char * const argv[]) +{ + int (*_execvp) (const char *, char * const argv[]) = dlsym(RTLD_NEXT, "execvp"); + char buf[PATH_MAX]; + return _execvp(rewrite(path, buf), argv); +} + +int execve(const char * path, char * const argv[], char * const envp[]) +{ + int (*_execve) (const char *, char * const argv[], char * const envp[]) = dlsym(RTLD_NEXT, "execve"); + char buf[PATH_MAX]; + return _execve(rewrite(path, buf), argv, envp); +} + +DIR * opendir(const char * path) +{ + char buf[PATH_MAX]; + DIR * (*_opendir) (const char*) = dlsym(RTLD_NEXT, "opendir"); + + return _opendir(rewrite(path, buf)); +} + +#define SYSTEM_CMD_MAX 512 + +char *replace_substring(char * source, char * buf, char * replace_string, char * start_ptr, char * suffix_ptr) { + char head[SYSTEM_CMD_MAX] = {0}; + strncpy(head, source, start_ptr - source); + + char tail[SYSTEM_CMD_MAX] = {0}; + if(suffix_ptr < source + strlen(source)) { + strcpy(tail, suffix_ptr); + } + + sprintf(buf, "%s%s%s", head, replace_string, tail); + return buf; +} + +char *replace_string(char * buf, char * from, char * to) { + int num_matches = 0; + char * matches[SYSTEM_CMD_MAX]; + int from_len = strlen(from); + for(int i=0; i<strlen(buf); i++){ + char *cmp_start = buf + i; + if(strncmp(from, cmp_start, from_len) == 0){ + matches[num_matches] = cmp_start; + num_matches++; + } + } + int len_diff = strlen(to) - strlen(from); + for(int n = 0; n < num_matches; n++) { + char replaced[SYSTEM_CMD_MAX]; + replace_substring(buf, replaced, to, matches[n], matches[n]+from_len); + strcpy(buf, replaced); + for(int nn = n+1; nn < num_matches; nn++) { + matches[nn] += len_diff; + } + } + return buf; +} + +void rewriteSystemCall(const char * command, char * buf) { + strcpy(buf, command); + for (int n = 0; n < nrRedirects; ++n) { + replace_string(buf, from[n], to[n]); + } +} + +int system(const char *command) +{ + int (*_system) (const char*) = dlsym(RTLD_NEXT, "system"); + + char newCommand[SYSTEM_CMD_MAX]; + rewriteSystemCall(command, newCommand); + return _system(newCommand); +} diff --git a/pkgs/build-support/libredirect/test.c b/pkgs/build-support/libredirect/test.c index b57664db3c1..722d1303771 100644 --- a/pkgs/build-support/libredirect/test.c +++ b/pkgs/build-support/libredirect/test.c @@ -2,6 +2,7 @@ #include <fcntl.h> #include <spawn.h> #include <stdio.h> +#include <stdlib.h> #include <unistd.h> #include <sys/stat.h> @@ -31,6 +32,10 @@ void test_execv(void) { assert(execv(TESTPATH, argv) == 0); } +void test_system(void) { + assert(system(TESTPATH) == 0); +} + int main(void) { FILE *testfp; @@ -50,6 +55,7 @@ int main(void) assert(stat(TESTPATH, &testsb) != -1); test_spawn(); + test_system(); test_execv(); /* If all goes well, this is never reached because test_execv() replaces diff --git a/pkgs/build-support/make-desktopitem/default.nix b/pkgs/build-support/make-desktopitem/default.nix index 8355a5ad29b..1491a3ad911 100644 --- a/pkgs/build-support/make-desktopitem/default.nix +++ b/pkgs/build-support/make-desktopitem/default.nix @@ -1,50 +1,67 @@ { lib, runCommandLocal, desktop-file-utils }: -{ name +# See https://specifications.freedesktop.org/desktop-entry-spec/desktop-entry-spec-latest.html +{ name # The name of the desktop file , type ? "Application" , exec , icon ? null , comment ? null -, terminal ? "false" -, desktopName +, terminal ? false +, desktopName # The name of the application , genericName ? null , mimeType ? null , categories ? null , startupNotify ? null -, extraEntries ? null +, noDisplay ? null +, prefersNonDefaultGPU ? null +, extraDesktopEntries ? { } # Extra key-value pairs to add to the [Desktop Entry] section. This may override other values +, extraEntries ? "" # Extra configuration. Will be appended to the end of the file and may thus contain extra sections , fileValidation ? true # whether to validate resulting desktop file. }: - let - optionalEntriesList = [{k="Icon"; v=icon;} - {k="Comment"; v=comment;} - {k="GenericName"; v=genericName;} - {k="MimeType"; v=mimeType;} - {k="Categories"; v=categories;} - {k="StartupNotify"; v=startupNotify;}]; + # like builtins.toString, but null -> null instead of null -> "" + nullableToString = value: + if value == null then null + else if builtins.isBool value then lib.boolToString value + else builtins.toString value; - valueNotNull = {k, v}: v != null; - entriesToKeep = builtins.filter valueNotNull optionalEntriesList; + # The [Desktop entry] section of the desktop file, as attribute set. + mainSection = { + "Type" = toString type; + "Exec" = nullableToString exec; + "Icon" = nullableToString icon; + "Comment" = nullableToString comment; + "Terminal" = nullableToString terminal; + "Name" = toString desktopName; + "GenericName" = nullableToString genericName; + "MimeType" = nullableToString mimeType; + "Categories" = nullableToString categories; + "StartupNotify" = nullableToString startupNotify; + "NoDisplay" = nullableToString noDisplay; + "PrefersNonDefaultGPU" = nullableToString prefersNonDefaultGPU; + } // extraDesktopEntries; - mkEntry = {k, v}: k + "=" + v; - optionalEntriesString = lib.concatMapStringsSep "\n" mkEntry entriesToKeep; + # Map all entries to a list of lines + desktopFileStrings = + [ "[Desktop Entry]" ] + ++ builtins.filter + (v: v != null) + (lib.mapAttrsToList + (name: value: if value != null then "${name}=${value}" else null) + mainSection + ) + ++ (if extraEntries == "" then [ ] else [ "${extraEntries}" ]); in -runCommandLocal "${name}.desktop" {} - '' +runCommandLocal "${name}.desktop" +{ + nativeBuildInputs = [ desktop-file-utils ]; +} + ('' mkdir -p "$out/share/applications" cat > "$out/share/applications/${name}.desktop" <<EOF - [Desktop Entry] - Type=${type} - Exec=${exec} - Terminal=${terminal} - Name=${desktopName} - ${optionalEntriesString} - ${if extraEntries == null then ''EOF'' else '' - ${extraEntries} - EOF''} - - ${lib.optionalString fileValidation '' - echo "Running desktop-file validation" - ${desktop-file-utils}/bin/desktop-file-validate "$out/share/applications/${name}.desktop" - ''} - '' + ${builtins.concatStringsSep "\n" desktopFileStrings} + EOF + '' + lib.optionalString fileValidation '' + echo "Running desktop-file validation" + desktop-file-validate "$out/share/applications/${name}.desktop" + '') diff --git a/pkgs/build-support/mkshell/default.nix b/pkgs/build-support/mkshell/default.nix index a70dc0390cb..7ca4cc23c1d 100644 --- a/pkgs/build-support/mkshell/default.nix +++ b/pkgs/build-support/mkshell/default.nix @@ -3,18 +3,22 @@ # A special kind of derivation that is only meant to be consumed by the # nix-shell. { - inputsFrom ? [], # a list of derivations whose inputs will be made available to the environment - buildInputs ? [], - nativeBuildInputs ? [], - propagatedBuildInputs ? [], - propagatedNativeBuildInputs ? [], - ... + # a list of packages to add to the shell environment + packages ? [ ] +, # propagate all the inputs from the given derivations + inputsFrom ? [ ] +, buildInputs ? [ ] +, nativeBuildInputs ? [ ] +, propagatedBuildInputs ? [ ] +, propagatedNativeBuildInputs ? [ ] +, ... }@attrs: let mergeInputs = name: lib.concatLists (lib.catAttrs name - ([attrs] ++ inputsFrom)); + ([ attrs ] ++ inputsFrom)); rest = builtins.removeAttrs attrs [ + "packages" "inputsFrom" "buildInputs" "nativeBuildInputs" @@ -26,15 +30,15 @@ in stdenv.mkDerivation ({ name = "nix-shell"; - phases = ["nobuildPhase"]; + phases = [ "nobuildPhase" ]; buildInputs = mergeInputs "buildInputs"; - nativeBuildInputs = mergeInputs "nativeBuildInputs"; + nativeBuildInputs = packages ++ (mergeInputs "nativeBuildInputs"); propagatedBuildInputs = mergeInputs "propagatedBuildInputs"; propagatedNativeBuildInputs = mergeInputs "propagatedNativeBuildInputs"; shellHook = lib.concatStringsSep "\n" (lib.catAttrs "shellHook" - (lib.reverseList inputsFrom ++ [attrs])); + (lib.reverseList inputsFrom ++ [ attrs ])); nobuildPhase = '' echo diff --git a/pkgs/build-support/nix-gitignore/default.nix b/pkgs/build-support/nix-gitignore/default.nix index 28ee6bad554..497dcb0660b 100644 --- a/pkgs/build-support/nix-gitignore/default.nix +++ b/pkgs/build-support/nix-gitignore/default.nix @@ -41,6 +41,9 @@ in rec { let split = match "^(!?)(.*)" l; in [(elemAt split 1) (head split == "!")]; + # regex -> regex + handleHashesBangs = replaceStrings ["\\#" "\\!"] ["#" "!"]; + # ignore -> regex substWildcards = let @@ -86,7 +89,7 @@ in rec { mapPat = f: l: [(f (head l)) (last l)]; in map (l: # `l' for "line" - mapPat (l: handleSlashSuffix (handleSlashPrefix (mapAroundCharclass substWildcards l))) + mapPat (l: handleSlashSuffix (handleSlashPrefix (handleHashesBangs (mapAroundCharclass substWildcards l)))) (computeNegation l)) (filter (l: !isList l && !isComment l) (split "\n" gitignore)); @@ -148,10 +151,10 @@ in rec { ''); withGitignoreFile = patterns: root: - lib.toList patterns ++ [(root + "/.gitignore")]; + lib.toList patterns ++ [ ".git" ] ++ [(root + "/.gitignore")]; withRecursiveGitignoreFile = patterns: root: - lib.toList patterns ++ [(compileRecursiveGitignore root)]; + lib.toList patterns ++ [ ".git" ] ++ [(compileRecursiveGitignore root)]; # filterSource derivatives diff --git a/pkgs/build-support/nuke-references/builder.sh b/pkgs/build-support/nuke-references/builder.sh deleted file mode 100644 index 7da32203218..00000000000 --- a/pkgs/build-support/nuke-references/builder.sh +++ /dev/null @@ -1,29 +0,0 @@ -source $stdenv/setup - -mkdir -p $out/bin -cat > $out/bin/nuke-refs <<EOF -#! $SHELL -e - -excludes="" -while getopts e: o; do - case "\$o" in - e) storeId=\$(echo "\$OPTARG" | $perl/bin/perl -ne "print \"\\\$1\" if m|^\Q$NIX_STORE\E/([a-z0-9]{32})-.*|") - if [ -z "\$storeId" ]; then - echo "-e argument must be a Nix store path" - exit 1 - fi - excludes="\$excludes(?!\$storeId)" - ;; - esac -done -shift \$((\$OPTIND-1)) - -for i in "\$@"; do - if test ! -L "\$i" -a -f "\$i"; then - cat "\$i" | $perl/bin/perl -pe "s|\Q$NIX_STORE\E/\$excludes[a-z0-9]{32}-|$NIX_STORE/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-|g" > "\$i.tmp" - if test -x "\$i"; then chmod +x "\$i.tmp"; fi - mv "\$i.tmp" "\$i" - fi -done -EOF -chmod +x $out/bin/nuke-refs diff --git a/pkgs/build-support/nuke-references/darwin-sign-fixup.sh b/pkgs/build-support/nuke-references/darwin-sign-fixup.sh new file mode 100644 index 00000000000..940c18e5a62 --- /dev/null +++ b/pkgs/build-support/nuke-references/darwin-sign-fixup.sh @@ -0,0 +1,5 @@ +# Fixup hook for nukeReferences, not stdenv + +source @signingUtils@ + +fixupHooks+=(signIfRequired) diff --git a/pkgs/build-support/nuke-references/default.nix b/pkgs/build-support/nuke-references/default.nix index d894b56d366..03f6fe53b54 100644 --- a/pkgs/build-support/nuke-references/default.nix +++ b/pkgs/build-support/nuke-references/default.nix @@ -3,11 +3,35 @@ # path (/nix/store/eeee...). This is useful for getting rid of # dependencies that you know are not actually needed at runtime. -{ stdenvNoCC, perl }: +{ lib, stdenvNoCC, perl, signingUtils, shell ? stdenvNoCC.shell }: + +let + stdenv = stdenvNoCC; + + darwinCodeSign = stdenv.targetPlatform.isDarwin && stdenv.targetPlatform.isAarch64; +in stdenvNoCC.mkDerivation { name = "nuke-references"; - builder = ./builder.sh; + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + + installPhase = '' + mkdir -p $out/bin + substituteAll ${./nuke-refs.sh} $out/bin/nuke-refs + chmod a+x $out/bin/nuke-refs + ''; + + postFixup = lib.optionalString darwinCodeSign '' + mkdir -p $out/nix-support + substituteAll ${./darwin-sign-fixup.sh} $out/nix-support/setup-hooks.sh + ''; + # FIXME: get rid of perl dependency. inherit perl; + inherit (builtins) storeDir; + shell = lib.getBin shell + (shell.shellPath or ""); + signingUtils = if darwinCodeSign then signingUtils else null; } diff --git a/pkgs/build-support/nuke-references/nuke-refs.sh b/pkgs/build-support/nuke-references/nuke-refs.sh new file mode 100644 index 00000000000..21eb855cbad --- /dev/null +++ b/pkgs/build-support/nuke-references/nuke-refs.sh @@ -0,0 +1,33 @@ +#! @shell@ + +fixupHooks=() + +if [ -e @out@/nix-support/setup-hooks.sh ]; then + source @out@/nix-support/setup-hooks.sh +fi + +excludes="" +while getopts e: o; do + case "$o" in + e) storeId=$(echo "$OPTARG" | @perl@/bin/perl -ne "print \"\$1\" if m|^\Q@storeDir@\E/([a-z0-9]{32})-.*|") + if [ -z "$storeId" ]; then + echo "-e argument must be a Nix store path" + exit 1 + fi + excludes="$excludes(?!$storeId)" + ;; + esac +done +shift $(($OPTIND-1)) + +for i in "$@"; do + if test ! -L "$i" -a -f "$i"; then + cat "$i" | @perl@/bin/perl -pe "s|\Q@storeDir@\E/$excludes[a-z0-9]{32}-|@storeDir@/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-|g" > "$i.tmp" + if test -x "$i"; then chmod +x "$i.tmp"; fi + mv "$i.tmp" "$i" + + for hook in "${fixupHooks[@]}"; do + eval "$hook" "$i" + done + fi +done diff --git a/pkgs/build-support/ocaml/default.nix b/pkgs/build-support/ocaml/default.nix index 3957b955a2c..88ed3dfc2c2 100644 --- a/pkgs/build-support/ocaml/default.nix +++ b/pkgs/build-support/ocaml/default.nix @@ -1,4 +1,4 @@ -{ stdenv, writeText, ocaml, findlib, ocamlbuild, camlp4 }: +{ lib, stdenv, writeText, ocaml, findlib, ocamlbuild, camlp4 }: { name, version, buildInputs ? [], createFindlibDestdir ? true, @@ -14,7 +14,7 @@ let }; in assert minimumSupportedOcamlVersion != null -> - stdenv.lib.versionOlder minimumSupportedOcamlVersion ocaml.version; + lib.versionOlder minimumSupportedOcamlVersion ocaml.version; stdenv.mkDerivation (args // { name = "ocaml-${name}-${version}"; diff --git a/pkgs/build-support/ocaml/dune.nix b/pkgs/build-support/ocaml/dune.nix index b134effab8a..c049878d013 100644 --- a/pkgs/build-support/ocaml/dune.nix +++ b/pkgs/build-support/ocaml/dune.nix @@ -1,11 +1,11 @@ -{ stdenv, ocaml, findlib, dune, dune_2, opaline }: +{ lib, stdenv, ocaml, findlib, dune_1, dune_2 }: { pname, version, buildInputs ? [], enableParallelBuilding ? true, ... }@args: -let Dune = if args.useDune2 or false then dune_2 else dune; in +let Dune = if args.useDune2 or false then dune_2 else dune_1; in -if args ? minimumOCamlVersion && - ! stdenv.lib.versionAtLeast ocaml.version args.minimumOCamlVersion +if (args ? minimumOCamlVersion && ! lib.versionAtLeast ocaml.version args.minimumOCamlVersion) || + (args ? minimalOCamlVersion && ! lib.versionAtLeast ocaml.version args.minimalOCamlVersion) then throw "${pname}-${version} is not available for OCaml ${ocaml.version}" else @@ -25,11 +25,11 @@ stdenv.mkDerivation ({ ''; installPhase = '' runHook preInstall - ${opaline}/bin/opaline -prefix $out -libdir $OCAMLFIND_DESTDIR + dune install --prefix $out --libdir $OCAMLFIND_DESTDIR ${pname} runHook postInstall ''; -} // args // { +} // (builtins.removeAttrs args [ "minimalOCamlVersion" ]) // { name = "ocaml${ocaml.version}-${pname}-${version}"; diff --git a/pkgs/build-support/ocaml/oasis.nix b/pkgs/build-support/ocaml/oasis.nix index c1d1d699765..ee231a6e258 100644 --- a/pkgs/build-support/ocaml/oasis.nix +++ b/pkgs/build-support/ocaml/oasis.nix @@ -1,4 +1,4 @@ -{ stdenv, ocaml_oasis, ocaml, findlib, ocamlbuild }: +{ lib, stdenv, ocaml_oasis, ocaml, findlib, ocamlbuild }: { pname, version, buildInputs ? [], meta ? { platforms = ocaml.meta.platforms or []; }, minimumOCamlVersion ? null, @@ -8,7 +8,7 @@ }@args: if args ? minimumOCamlVersion && - ! stdenv.lib.versionAtLeast ocaml.version args.minimumOCamlVersion + ! lib.versionAtLeast ocaml.version args.minimumOCamlVersion then throw "${pname}-${version} is not available for OCaml ${ocaml.version}" else diff --git a/pkgs/build-support/pkg-config-wrapper/add-flags.sh b/pkgs/build-support/pkg-config-wrapper/add-flags.sh index 35ecf62ca23..90aee712be6 100644 --- a/pkgs/build-support/pkg-config-wrapper/add-flags.sh +++ b/pkgs/build-support/pkg-config-wrapper/add-flags.sh @@ -6,7 +6,7 @@ var_templates_list=( accumulateRoles for var in "${var_templates_list[@]}"; do - mangleVarList "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} + mangleVarListGeneric ":" "$var" ${role_suffixes[@]+"${role_suffixes[@]}"} done export NIX_PKG_CONFIG_WRAPPER_FLAGS_SET_@suffixSalt@=1 diff --git a/pkgs/build-support/pkg-config-wrapper/default.nix b/pkgs/build-support/pkg-config-wrapper/default.nix index e01df107dd1..bbc49d6728c 100644 --- a/pkgs/build-support/pkg-config-wrapper/default.nix +++ b/pkgs/build-support/pkg-config-wrapper/default.nix @@ -2,6 +2,7 @@ # PKG_CONFIG_PATH_FOR_BUILD work properly. { stdenvNoCC +, lib , buildPackages , pkg-config , baseBinName ? "pkg-config" @@ -9,7 +10,7 @@ , extraPackages ? [], extraBuildCommands ? "" }: -with stdenvNoCC.lib; +with lib; let stdenv = stdenvNoCC; @@ -19,7 +20,7 @@ let # # TODO(@Ericson2314) Make unconditional, or optional but always true by # default. - targetPrefix = stdenv.lib.optionalString (targetPlatform != hostPlatform) + targetPrefix = lib.optionalString (targetPlatform != hostPlatform) (targetPlatform.config + "-"); # See description in cc-wrapper. @@ -119,7 +120,7 @@ stdenv.mkDerivation { let pkg-config_ = if pkg-config != null then pkg-config else {}; in (if pkg-config_ ? meta then removeAttrs pkg-config.meta ["priority"] else {}) // { description = - stdenv.lib.attrByPath ["meta" "description"] "pkg-config" pkg-config_ + lib.attrByPath ["meta" "description"] "pkg-config" pkg-config_ + " (wrapper script)"; priority = 10; }; diff --git a/pkgs/build-support/plugins.nix b/pkgs/build-support/plugins.nix index bf8a982a88f..31b478c6c0d 100644 --- a/pkgs/build-support/plugins.nix +++ b/pkgs/build-support/plugins.nix @@ -1,4 +1,4 @@ -{ stdenv }: +{ lib }: # helper functions for packaging programs with plugin systems { @@ -13,7 +13,7 @@ diffPlugins = expectedPlugins: foundPluginsFilePath: '' # sort both lists first plugins_expected=$(mktemp) - (${stdenv.lib.concatMapStrings (s: "echo \"${s}\";") expectedPlugins}) \ + (${lib.concatMapStrings (s: "echo \"${s}\";") expectedPlugins}) \ | sort -u > "$plugins_expected" plugins_found=$(mktemp) sort -u "${foundPluginsFilePath}" > "$plugins_found" diff --git a/pkgs/build-support/release/ant-build.nix b/pkgs/build-support/release/ant-build.nix index 2d24d5bd704..6b59241e01e 100644 --- a/pkgs/build-support/release/ant-build.nix +++ b/pkgs/build-support/release/ant-build.nix @@ -1,5 +1,6 @@ { src , pkgs +, lib , stdenv ? pkgs.stdenv , name , antTargets ? [] @@ -16,8 +17,7 @@ , ... } @ args: let - antFlags = "-f ${buildfile} " + stdenv.lib.concatMapStrings ({name, value}: "-D${name}=${value} " ) antProperties ; - lib = stdenv.lib; + antFlags = "-f ${buildfile} " + lib.concatMapStrings ({name, value}: "-D${name}=${value} " ) antProperties ; in stdenv.mkDerivation ( @@ -31,7 +31,7 @@ stdenv.mkDerivation ( prePhases = ["antSetupPhase"]; - antSetupPhase = with stdenv.lib; '' + antSetupPhase = with lib; '' if test "$hydraAntLogger" != "" ; then export ANT_ARGS="-logger org.hydra.ant.HydraLogger -lib `ls $hydraAntLogger/share/java/*.jar | head -1`" fi @@ -46,7 +46,7 @@ stdenv.mkDerivation ( mkdir -p $out/share/java ${ if jars == [] then '' find . -name "*.jar" | xargs -I{} cp -v {} $out/share/java - '' else stdenv.lib.concatMapStrings (j: '' + '' else lib.concatMapStrings (j: '' cp -v ${j} $out/share/java '') jars } @@ -65,7 +65,7 @@ stdenv.mkDerivation ( in '' header "Generating jar wrappers" - '' + (stdenv.lib.concatMapStrings (w: '' + '' + (lib.concatMapStrings (w: '' mkdir -p $out/bin cat >> $out/bin/${w.name} <<EOF @@ -85,7 +85,7 @@ stdenv.mkDerivation ( header "Building default ant target" ant ${antFlags} closeNest - '' else stdenv.lib.concatMapStrings (t: '' + '' else lib.concatMapStrings (t: '' header "Building '${t}' target" ant ${antFlags} ${t} closeNest @@ -103,12 +103,13 @@ stdenv.mkDerivation ( ''; } - // removeAttrs args ["antProperties" "buildInputs" "pkgs" "jarWrappers"] // + // removeAttrs args ["antProperties" "buildInputs" "pkgs" "lib" "jarWrappers"] // { name = name + (if src ? version then "-" + src.version else ""); - buildInputs = [ant jre zip unzip] ++ stdenv.lib.optional (args ? buildInputs) args.buildInputs ; + nativeBuildInputs = [ unzip ]; + buildInputs = [ant jre zip] ++ lib.optional (args ? buildInputs) args.buildInputs ; postHook = '' mkdir -p $out/nix-support diff --git a/pkgs/build-support/release/debian-build.nix b/pkgs/build-support/release/debian-build.nix index dfa896a86a6..bd54401e235 100644 --- a/pkgs/build-support/release/debian-build.nix +++ b/pkgs/build-support/release/debian-build.nix @@ -11,8 +11,6 @@ debRequires ? [] , ... } @ args: -with stdenv.lib; - vmTools.runInLinuxImage (stdenv.mkDerivation ( { @@ -23,7 +21,7 @@ vmTools.runInLinuxImage (stdenv.mkDerivation ( prePhases = "installExtraDebsPhase sysInfoPhase"; } - // removeAttrs args ["vmTools"] // + // removeAttrs args ["vmTools" "lib"] // { name = name + "-" + diskImage.name + (if src ? version then "-" + src.version else ""); @@ -59,8 +57,8 @@ vmTools.runInLinuxImage (stdenv.mkDerivation ( export PAGER=cat ${checkinstall}/sbin/checkinstall --nodoc -y -D \ --fstrans=${if fsTranslation then "yes" else "no"} \ - --requires="${concatStringsSep "," debRequires}" \ - --provides="${concatStringsSep "," debProvides}" \ + --requires="${lib.concatStringsSep "," debRequires}" \ + --provides="${lib.concatStringsSep "," debProvides}" \ ${if (src ? version) then "--pkgversion=$(echo ${src.version} | tr _ -)" else "--pkgversion=0.0.0"} \ ''${debMaintainer:+--maintainer="'$debMaintainer'"} \ diff --git a/pkgs/build-support/release/default.nix b/pkgs/build-support/release/default.nix index 6b9aa9a8c4a..83f755b2bec 100644 --- a/pkgs/build-support/release/default.nix +++ b/pkgs/build-support/release/default.nix @@ -1,4 +1,4 @@ -{ pkgs }: +{ lib, pkgs }: with pkgs; @@ -15,7 +15,7 @@ rec { } // args); antBuild = args: import ./ant-build.nix ( - { inherit pkgs; + { inherit lib pkgs; } // args); mvnBuild = args: import ./maven-build.nix ( @@ -23,7 +23,7 @@ rec { } // args); nixBuild = args: import ./nix-build.nix ( - { inherit stdenv; + { inherit lib stdenv; } // args); coverageAnalysis = args: nixBuild ( @@ -41,16 +41,12 @@ rec { doCoverityAnalysis = true; } // args); - gcovReport = args: import ./gcov-report.nix ( - { inherit runCommand lcov rsync; - } // args); - rpmBuild = args: import ./rpm-build.nix ( { inherit vmTools; } // args); debBuild = args: import ./debian-build.nix ( - { inherit stdenv vmTools checkinstall; + { inherit lib stdenv vmTools checkinstall; } // args); aggregate = @@ -98,7 +94,7 @@ rec { phases = [ "unpackPhase" "patchPhase" "installPhase" ]; - patchPhase = stdenv.lib.optionalString isNixOS '' + patchPhase = lib.optionalString isNixOS '' touch .update-on-nixos-rebuild ''; diff --git a/pkgs/build-support/release/gcov-report.nix b/pkgs/build-support/release/gcov-report.nix deleted file mode 100644 index 8ce5c0488a7..00000000000 --- a/pkgs/build-support/release/gcov-report.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ runCommand, lcov, rsync, coverageRuns, lcovFilter ? [ "/nix/store/*" ], baseDirHack ? false }: - -runCommand "coverage" - { buildInputs = [ lcov rsync ]; - inherit lcovFilter baseDirHack; - } - '' - mkdir -p $TMPDIR/gcov $out/nix-support $out/coverage - info=$out/coverage/full.info - - for p in ${toString coverageRuns}; do - if [ -f $p/nix-support/hydra-build-products ]; then - cat $p/nix-support/hydra-build-products >> $out/nix-support/hydra-build-products - fi - - [ ! -e $p/nix-support/failed ] || touch $out/nix-support/failed - - opts= - for d in $p/coverage-data/*; do - for i in $(cd $d/nix/store && ls); do - if ! [ -e /nix/store/$i/.build ]; then continue; fi - if [ -e $TMPDIR/gcov/nix/store/$i ]; then continue; fi - echo "copying $i..." - rsync -a /nix/store/$i/.build/* $TMPDIR/gcov/ - if [ -n "$baseDirHack" ]; then - opts="-b $TMPDIR/gcov/$(cd /nix/store/$i/.build && ls)" - fi - done - - for i in $(cd $d/nix/store && ls); do - rsync -a $d/nix/store/$i/.build/* $TMPDIR/gcov/ --include '*/' --include '*.gcda' --exclude '*' - done - done - - chmod -R u+w $TMPDIR/gcov - - echo "producing info..." - geninfo --ignore-errors source,gcov $TMPDIR/gcov --output-file $TMPDIR/app.info $opts - cat $TMPDIR/app.info >> $info - done - - echo "making report..." - set -o noglob - lcov --remove $info ''$lcovFilter > $info.tmp - set +o noglob - mv $info.tmp $info - genhtml --show-details $info -o $out/coverage - echo "report coverage $out/coverage" >> $out/nix-support/hydra-build-products - '' diff --git a/pkgs/build-support/release/nix-build.nix b/pkgs/build-support/release/nix-build.nix index feda54de46f..ac51b90e016 100644 --- a/pkgs/build-support/release/nix-build.nix +++ b/pkgs/build-support/release/nix-build.nix @@ -12,7 +12,7 @@ , doCoverityAnalysis ? false , lcovFilter ? [] , lcovExtraTraceFiles ? [] -, src, stdenv +, src, lib, stdenv , name ? if doCoverageAnalysis then "nix-coverage" else "nix-build" , failureHook ? null , prePhases ? [] @@ -69,7 +69,7 @@ stdenv.mkDerivation ( fi ''; - failureHook = (stdenv.lib.optionalString (failureHook != null) failureHook) + + failureHook = (lib.optionalString (failureHook != null) failureHook) + '' if test -n "$succeedOnFailure"; then if test -n "$keepBuildDirectory"; then @@ -83,9 +83,9 @@ stdenv.mkDerivation ( ''; } - // args // + // removeAttrs args [ "lib" ] # Propagating lib causes the evaluation to fail, because lib is a function that can't be converted to a string - { + // { name = name + (if src ? version then "-" + src.version else ""); postHook = '' @@ -136,10 +136,10 @@ stdenv.mkDerivation ( buildInputs = buildInputs ++ - (stdenv.lib.optional doCoverageAnalysis args.makeGCOVReport) ++ - (stdenv.lib.optional doClangAnalysis args.clang-analyzer) ++ - (stdenv.lib.optional doCoverityAnalysis args.cov-build) ++ - (stdenv.lib.optional doCoverityAnalysis args.xz); + (lib.optional doCoverageAnalysis args.makeGCOVReport) ++ + (lib.optional doClangAnalysis args.clang-analyzer) ++ + (lib.optional doCoverityAnalysis args.cov-build) ++ + (lib.optional doCoverityAnalysis args.xz); lcovFilter = ["/nix/store/*"] ++ lcovFilter; diff --git a/pkgs/build-support/remove-references-to/darwin-sign-fixup.sh b/pkgs/build-support/remove-references-to/darwin-sign-fixup.sh new file mode 100644 index 00000000000..940c18e5a62 --- /dev/null +++ b/pkgs/build-support/remove-references-to/darwin-sign-fixup.sh @@ -0,0 +1,5 @@ +# Fixup hook for nukeReferences, not stdenv + +source @signingUtils@ + +fixupHooks+=(signIfRequired) diff --git a/pkgs/build-support/remove-references-to/default.nix b/pkgs/build-support/remove-references-to/default.nix index 8b1d05fc230..f022611ef91 100644 --- a/pkgs/build-support/remove-references-to/default.nix +++ b/pkgs/build-support/remove-references-to/default.nix @@ -3,32 +3,33 @@ # non-existent path (/nix/store/eeee...). This is useful for getting rid of # dependencies that you know are not actually needed at runtime. -{ stdenv, writeScriptBin }: - -writeScriptBin "remove-references-to" '' -#! ${stdenv.shell} -e - -# References to remove -targets=() -while getopts t: o; do - case "$o" in - t) storeId=$(echo "$OPTARG" | sed -n "s|^$NIX_STORE/\\([a-z0-9]\{32\}\\)-.*|\1|p") - if [ -z "$storeId" ]; then - echo "-t argument must be a Nix store path" - exit 1 - fi - targets+=("$storeId") - esac -done -shift $(($OPTIND-1)) - -# Files to remove the references from -regions=() -for i in "$@"; do - test ! -L "$i" -a -f "$i" && regions+=("$i") -done - -for target in "''${targets[@]}" ; do - sed -i -e "s|$NIX_STORE/$target-|$NIX_STORE/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-|g" "''${regions[@]}" -done -'' +{ lib, stdenvNoCC, signingUtils, shell ? stdenvNoCC.shell }: + +let + stdenv = stdenvNoCC; + + darwinCodeSign = stdenv.targetPlatform.isDarwin && stdenv.targetPlatform.isAarch64; +in + +stdenv.mkDerivation { + name = "remove-references-to"; + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + + installPhase = '' + mkdir -p $out/bin + substituteAll ${./remove-references-to.sh} $out/bin/remove-references-to + chmod a+x $out/bin/remove-references-to + ''; + + postFixup = lib.optionalString darwinCodeSign '' + mkdir -p $out/nix-support + substituteAll ${./darwin-sign-fixup.sh} $out/nix-support/setup-hooks.sh + ''; + + inherit (builtins) storeDir; + shell = lib.getBin shell + (shell.shellPath or ""); + signingUtils = if darwinCodeSign then signingUtils else null; +} diff --git a/pkgs/build-support/remove-references-to/remove-references-to.sh b/pkgs/build-support/remove-references-to/remove-references-to.sh new file mode 100644 index 00000000000..d8d38dbd80a --- /dev/null +++ b/pkgs/build-support/remove-references-to/remove-references-to.sh @@ -0,0 +1,37 @@ +#! @shell@ -e + +fixupHooks=() + +if [ -e @out@/nix-support/setup-hooks.sh ]; then + source @out@/nix-support/setup-hooks.sh +fi + +# References to remove +targets=() +while getopts t: o; do + case "$o" in + t) storeId=$(echo "$OPTARG" | sed -n "s|^@storeDir@/\\([a-z0-9]\{32\}\\)-.*|\1|p") + if [ -z "$storeId" ]; then + echo "-t argument must be a Nix store path" + exit 1 + fi + targets+=("$storeId") + esac +done +shift $(($OPTIND-1)) + +# Files to remove the references from +regions=() +for i in "$@"; do + test ! -L "$i" -a -f "$i" && regions+=("$i") +done + +for target in "${targets[@]}" ; do + sed -i -e "s|@storeDir@/$target-|@storeDir@/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-|g" "${regions[@]}" +done + +for region in "${regions[@]}"; do + for hook in "${fixupHooks[@]}"; do + eval "$hook" "$i" + done +done diff --git a/pkgs/build-support/replace-secret/replace-secret.nix b/pkgs/build-support/replace-secret/replace-secret.nix new file mode 100644 index 00000000000..e04d1aed5f7 --- /dev/null +++ b/pkgs/build-support/replace-secret/replace-secret.nix @@ -0,0 +1,35 @@ +{ stdenv, lib, python3 }: + +stdenv.mkDerivation { + name = "replace-secret"; + buildInputs = [ python3 ]; + phases = [ "installPhase" "checkPhase" ]; + installPhase = '' + install -D ${./replace-secret.py} $out/bin/replace-secret + patchShebangs $out + ''; + doCheck = true; + checkPhase = '' + install -m 0600 ${./test/input_file} long_test + $out/bin/replace-secret "replace this" ${./test/passwd} long_test + $out/bin/replace-secret "and this" ${./test/rsa} long_test + diff ${./test/expected_long_output} long_test + + install -m 0600 ${./test/input_file} short_test + $out/bin/replace-secret "replace this" <(echo "a") short_test + $out/bin/replace-secret "and this" <(echo "b") short_test + diff ${./test/expected_short_output} short_test + ''; + meta = with lib; { + platforms = platforms.all; + maintainers = with maintainers; [ talyz ]; + license = licenses.mit; + description = "Replace a string in one file with a secret from a second file"; + longDescription = '' + Replace a string in one file with a secret from a second file. + + Since the secret is read from a file, it won't be leaked through + '/proc/<pid>/cmdline', unlike when 'sed' or 'replace' is used. + ''; + }; +} diff --git a/pkgs/build-support/replace-secret/replace-secret.py b/pkgs/build-support/replace-secret/replace-secret.py new file mode 100755 index 00000000000..30ff41d491b --- /dev/null +++ b/pkgs/build-support/replace-secret/replace-secret.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +import argparse +from argparse import RawDescriptionHelpFormatter + +description = """ +Replace a string in one file with a secret from a second file. + +Since the secret is read from a file, it won't be leaked through +'/proc/<pid>/cmdline', unlike when 'sed' or 'replace' is used. +""" + +parser = argparse.ArgumentParser( + description=description, + formatter_class=RawDescriptionHelpFormatter +) +parser.add_argument("string_to_replace", help="the string to replace") +parser.add_argument("secret_file", help="the file containing the secret") +parser.add_argument("file", help="the file to perform the replacement on") +args = parser.parse_args() + +with open(args.secret_file) as sf, open(args.file, 'r+') as f: + old = f.read() + secret = sf.read().strip("\n") + new_content = old.replace(args.string_to_replace, secret) + f.seek(0) + f.write(new_content) + f.truncate() diff --git a/pkgs/build-support/replace-secret/test/expected_long_output b/pkgs/build-support/replace-secret/test/expected_long_output new file mode 100644 index 00000000000..37bd66b905f --- /dev/null +++ b/pkgs/build-support/replace-secret/test/expected_long_output @@ -0,0 +1,30 @@ +beginning +middle $6$UcbJUl5g$HRMfKNKsLTfVbcQb.P5o0bmZUfHDYkWseMSuZ8F5jSIGZZcI3Jnit23f8ZeZOGi4KL86HVM9RYqrpYySOu/fl0 not this +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAzrru6v5tfwQl6L+rOUjtLo8kbhMUlCLXP7TYngSGrkzPMWe+ +0gB04UAmiPZXfBmvj5fPqYiFjIaEDHE/SD41vJB/RJKKtId2gCAIHhBLkbr+4+60 +yEbLkJci5i4kJC1dt8OKFEzXkaVnwOSgjH+0NwO3bstZ+E70zMXS9+NS71qGsIEb +5J1TnacwW/u6CdFyakLljWOXOR14rLIpiPBBFLf+oZiepjIhlWXWHqsxZOb7zMI0 +T4W5WJ2dwGFsJ8rkYaGZ+A5qzYbi/KmHqaSPaNDsyoi7yJhAhKPByALJU916+8QO +xOnqZxWGki3PDzCslRwW4i3mGbZlBQMnlfbN3QIDAQABAoIBAHDn1W7QkFrLmCy6 +6bf6pVdFZF8d2qJhOPAZRClhTXFKj+pqv+QPzcXr9F/fMr6bhK/G+Oqdnlq2aM4m +16oMF+spe+impEyeo1CsreJFghBQcb9o8qFjUPBiKvROBP0hLcscZ4BYy29HSBgo +harWYEWfqQJA251q+fYQoP0z0WrZKddOZbRRnJ0ICRxAE7IEtDT6EYt8R9oGi2j4 +/rpdW+rYGjW3TcmzdR7lpVMJRLlbMbSdR8n6cI6rnfySygcoE5tFX5t/YZSNbBPg +GebKCbEHYNTTG8bC1qjUyzlbEQ6XYWvFO7HTKU7105XpjYTQFByeo0IVkin0o5KW +t7eQWb0CgYEA6zZUWsYoQ13nXEU6Ky89Q9uhesMfaJ/F2X5ikQSRqRvrR3QR+ULe +eNnCl10O9SiFpR4b5gSbLSHMffxGN60P1nEO4CiIKE+gOii8Kdk5htIJFy/dcZUc +PuPM+zD9/6Is5sAWUZo45bnT6685h6EjM2+6zNZtx/XMjSfWbHaY+HMCgYEA4QAy +6ZEgd6FHnNfM/q2o8XU3d6OCdhcu26u6ydnCalbSpPSKWOi6gnHK4ZnGdryXgIYw +hRkvYINfiONkShYytotIh4YxUbgpwdvJRyKa2ZdWhcMmtFzZOcEVzQTKBasFT74C +Wo0iybZ++XZh3M0+n7oyyx39aR7diZ+/zq6PnG8CgYB8B1QH4cHNdDDRqPd5WhmW +NLQ7xbREOSvc+hYDnkMoxz4TmZL4u1gQpdNEeZ+visSeQvg3HGqvK8lnDaYBKdLW +IxvS+8yAZSx6PoyqDI+XFh4RCf5dLGGOkBTAyB7Hs761lsiuEwK5sHmdJ/LQIBot +v1bjOJb/AA/yxvT8kLUtHQKBgGIA9iwqXJv/EfRNQytDdS0HQ4vHGtJZMr3YRVoa +kcZD3yieo4wqguLCsf4mPv4FE3CWAphW6f39+yTi9xIWLSy56nOtjdnsf7PDCh8E +AbL5amSFJly1fKDda6OLjHt/jKa5Osk6ZIa8CP6cA/BrLfXg4rL6cyDQouqJPMDH +5CHdAoGBAIChjbTyoYvANkoANCK4SuqLUYeiYREfiM3sqHe1xirK1PPHw03ZLITl +ltjo9qE6kPXWcTBVckTKGFlntyCT283FC0/vMmHo8dTdtxF4/wSbkqs3ORuJ3p5J +cNtLYGD3vgwLmg6tTur4U60XN+tYDzWGteez8J9GwTMfKJmuS9af +-----END RSA PRIVATE KEY----- +end diff --git a/pkgs/build-support/replace-secret/test/expected_short_output b/pkgs/build-support/replace-secret/test/expected_short_output new file mode 100644 index 00000000000..3c81b2e2f99 --- /dev/null +++ b/pkgs/build-support/replace-secret/test/expected_short_output @@ -0,0 +1,4 @@ +beginning +middle a not this +b +end diff --git a/pkgs/build-support/replace-secret/test/input_file b/pkgs/build-support/replace-secret/test/input_file new file mode 100644 index 00000000000..1e7eadfaab2 --- /dev/null +++ b/pkgs/build-support/replace-secret/test/input_file @@ -0,0 +1,4 @@ +beginning +middle replace this not this +and this +end diff --git a/pkgs/build-support/replace-secret/test/passwd b/pkgs/build-support/replace-secret/test/passwd new file mode 100644 index 00000000000..68f266226e4 --- /dev/null +++ b/pkgs/build-support/replace-secret/test/passwd @@ -0,0 +1 @@ +$6$UcbJUl5g$HRMfKNKsLTfVbcQb.P5o0bmZUfHDYkWseMSuZ8F5jSIGZZcI3Jnit23f8ZeZOGi4KL86HVM9RYqrpYySOu/fl0 diff --git a/pkgs/build-support/replace-secret/test/rsa b/pkgs/build-support/replace-secret/test/rsa new file mode 100644 index 00000000000..138cc99ed22 --- /dev/null +++ b/pkgs/build-support/replace-secret/test/rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAzrru6v5tfwQl6L+rOUjtLo8kbhMUlCLXP7TYngSGrkzPMWe+ +0gB04UAmiPZXfBmvj5fPqYiFjIaEDHE/SD41vJB/RJKKtId2gCAIHhBLkbr+4+60 +yEbLkJci5i4kJC1dt8OKFEzXkaVnwOSgjH+0NwO3bstZ+E70zMXS9+NS71qGsIEb +5J1TnacwW/u6CdFyakLljWOXOR14rLIpiPBBFLf+oZiepjIhlWXWHqsxZOb7zMI0 +T4W5WJ2dwGFsJ8rkYaGZ+A5qzYbi/KmHqaSPaNDsyoi7yJhAhKPByALJU916+8QO +xOnqZxWGki3PDzCslRwW4i3mGbZlBQMnlfbN3QIDAQABAoIBAHDn1W7QkFrLmCy6 +6bf6pVdFZF8d2qJhOPAZRClhTXFKj+pqv+QPzcXr9F/fMr6bhK/G+Oqdnlq2aM4m +16oMF+spe+impEyeo1CsreJFghBQcb9o8qFjUPBiKvROBP0hLcscZ4BYy29HSBgo +harWYEWfqQJA251q+fYQoP0z0WrZKddOZbRRnJ0ICRxAE7IEtDT6EYt8R9oGi2j4 +/rpdW+rYGjW3TcmzdR7lpVMJRLlbMbSdR8n6cI6rnfySygcoE5tFX5t/YZSNbBPg +GebKCbEHYNTTG8bC1qjUyzlbEQ6XYWvFO7HTKU7105XpjYTQFByeo0IVkin0o5KW +t7eQWb0CgYEA6zZUWsYoQ13nXEU6Ky89Q9uhesMfaJ/F2X5ikQSRqRvrR3QR+ULe +eNnCl10O9SiFpR4b5gSbLSHMffxGN60P1nEO4CiIKE+gOii8Kdk5htIJFy/dcZUc +PuPM+zD9/6Is5sAWUZo45bnT6685h6EjM2+6zNZtx/XMjSfWbHaY+HMCgYEA4QAy +6ZEgd6FHnNfM/q2o8XU3d6OCdhcu26u6ydnCalbSpPSKWOi6gnHK4ZnGdryXgIYw +hRkvYINfiONkShYytotIh4YxUbgpwdvJRyKa2ZdWhcMmtFzZOcEVzQTKBasFT74C +Wo0iybZ++XZh3M0+n7oyyx39aR7diZ+/zq6PnG8CgYB8B1QH4cHNdDDRqPd5WhmW +NLQ7xbREOSvc+hYDnkMoxz4TmZL4u1gQpdNEeZ+visSeQvg3HGqvK8lnDaYBKdLW +IxvS+8yAZSx6PoyqDI+XFh4RCf5dLGGOkBTAyB7Hs761lsiuEwK5sHmdJ/LQIBot +v1bjOJb/AA/yxvT8kLUtHQKBgGIA9iwqXJv/EfRNQytDdS0HQ4vHGtJZMr3YRVoa +kcZD3yieo4wqguLCsf4mPv4FE3CWAphW6f39+yTi9xIWLSy56nOtjdnsf7PDCh8E +AbL5amSFJly1fKDda6OLjHt/jKa5Osk6ZIa8CP6cA/BrLfXg4rL6cyDQouqJPMDH +5CHdAoGBAIChjbTyoYvANkoANCK4SuqLUYeiYREfiM3sqHe1xirK1PPHw03ZLITl +ltjo9qE6kPXWcTBVckTKGFlntyCT283FC0/vMmHo8dTdtxF4/wSbkqs3ORuJ3p5J +cNtLYGD3vgwLmg6tTur4U60XN+tYDzWGteez8J9GwTMfKJmuS9af +-----END RSA PRIVATE KEY----- diff --git a/pkgs/build-support/rust/build-rust-crate/build-crate.nix b/pkgs/build-support/rust/build-rust-crate/build-crate.nix index 142109cef49..3441e2c5e7b 100644 --- a/pkgs/build-support/rust/build-rust-crate/build-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/build-crate.nix @@ -9,13 +9,14 @@ let baseRustcOpts = - [(if release then "-C opt-level=3" else "-C debuginfo=2")] - ++ ["-C codegen-units=$NIX_BUILD_CORES"] - ++ ["--remap-path-prefix=$NIX_BUILD_TOP=/" ] - ++ [(mkRustcDepArgs dependencies crateRenames)] - ++ [(mkRustcFeatureArgs crateFeatures)] - ++ extraRustcOpts - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTarget stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc" + [ + (if release then "-C opt-level=3" else "-C debuginfo=2") + "-C codegen-units=$NIX_BUILD_CORES" + "--remap-path-prefix=$NIX_BUILD_TOP=/" + (mkRustcDepArgs dependencies crateRenames) + (mkRustcFeatureArgs crateFeatures) + ] ++ extraRustcOpts + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--target ${rust.toRustTargetSpec stdenv.hostPlatform} -C linker=${stdenv.hostPlatform.config}-gcc" # since rustc 1.42 the "proc_macro" crate is part of the default crate prelude # https://github.com/rust-lang/cargo/commit/4d64eb99a4#diff-7f98585dbf9d30aa100c8318e2c77e79R1021-R1022 ++ lib.optional (lib.elem "proc-macro" crateType) "--extern proc_macro" diff --git a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix index a95b356646e..d1010ac1adb 100644 --- a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: +{ lib, stdenv, rust, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: { build , buildDependencies @@ -17,7 +17,6 @@ , libName , libPath , release -, target_os , verbose , workspace_member }: let version_ = lib.splitString "-" crateVersion; @@ -124,8 +123,8 @@ in '' export CARGO_PKG_AUTHORS="${authors}" export CARGO_PKG_DESCRIPTION="${crateDescription}" - export CARGO_CFG_TARGET_ARCH=${stdenv.hostPlatform.parsed.cpu.name} - export CARGO_CFG_TARGET_OS=${target_os} + export CARGO_CFG_TARGET_ARCH=${rust.toTargetArch stdenv.hostPlatform} + export CARGO_CFG_TARGET_OS=${rust.toTargetOs stdenv.hostPlatform} export CARGO_CFG_TARGET_FAMILY="unix" export CARGO_CFG_UNIX=1 export CARGO_CFG_TARGET_ENV="gnu" @@ -136,8 +135,8 @@ in '' export CARGO_MANIFEST_DIR=$(pwd) export DEBUG="${toString (!release)}" export OPT_LEVEL="${toString optLevel}" - export TARGET="${stdenv.hostPlatform.config}" - export HOST="${stdenv.hostPlatform.config}" + export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}" + export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}" export PROFILE=${if release then "release" else "debug"} export OUT_DIR=$(pwd)/target/build/${crateName}.out export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0} @@ -145,7 +144,7 @@ in '' export CARGO_PKG_VERSION_PATCH=${lib.elemAt version 2} export CARGO_PKG_VERSION_PRE="${versionPre}" export CARGO_PKG_HOMEPAGE="${crateHomepage}" - export NUM_JOBS=1 + export NUM_JOBS=$NIX_BUILD_CORES export RUSTC="rustc" export RUSTDOC="rustdoc" diff --git a/pkgs/build-support/rust/build-rust-crate/default.nix b/pkgs/build-support/rust/build-rust-crate/default.nix index d559aba1616..e605c9550e5 100644 --- a/pkgs/build-support/rust/build-rust-crate/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/default.nix @@ -4,15 +4,10 @@ # This can be useful for deploying packages with NixOps, and to share # binary dependencies between projects. -{ lib, stdenv, defaultCrateOverrides, fetchCrate, rustc, rust, cargo, jq }: +{ lib, stdenv, defaultCrateOverrides, fetchCrate, pkgsBuildBuild, rustc, rust +, cargo, jq }: let - # This doesn't appear to be officially documented anywhere yet. - # See https://github.com/rust-lang-nursery/rust-forge/issues/101. - target_os = if stdenv.hostPlatform.isDarwin - then "macos" - else stdenv.hostPlatform.parsed.kernel.name; - # Create rustc arguments to link against the given list of dependencies # and renames. # @@ -51,7 +46,7 @@ let inherit (import ./log.nix { inherit lib; }) noisily echo_colored; configureCrate = import ./configure-crate.nix { - inherit lib stdenv echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; + inherit lib stdenv rust echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; }; buildCrate = import ./build-crate.nix { @@ -59,6 +54,10 @@ let }; installCrate = import ./install-crate.nix { inherit stdenv; }; + + # Allow access to the rust attribute set from inside buildRustCrate, which + # has a parameter that shadows the name. + rustAttrs = rust; in /* The overridable pkgs.buildRustCrate function. @@ -83,6 +82,8 @@ in # A list of rust/cargo features to enable while building the crate. # Example: [ "std" "async" ] , features + # Additional native build inputs for building this crate. + , nativeBuildInputs # Additional build inputs for building this crate. # # Example: [ pkgs.openssl ] @@ -188,12 +189,13 @@ let crate = crate_ // (lib.attrByPath [ crate_.crateName ] (attr: {}) crateOverr dependencies_ = dependencies; buildDependencies_ = buildDependencies; processedAttrs = [ - "src" "buildInputs" "crateBin" "crateLib" "libName" "libPath" + "src" "nativeBuildInputs" "buildInputs" "crateBin" "crateLib" "libName" "libPath" "buildDependencies" "dependencies" "features" "crateRenames" "crateName" "version" "build" "authors" "colors" "edition" "buildTests" ]; extraDerivationAttrs = builtins.removeAttrs crate processedAttrs; + nativeBuildInputs_ = nativeBuildInputs; buildInputs_ = buildInputs; extraRustcOpts_ = extraRustcOpts; buildTests_ = buildTests; @@ -225,7 +227,8 @@ stdenv.mkDerivation (rec { src = crate.src or (fetchCrate { inherit (crate) crateName version sha256; }); name = "rust_${crate.crateName}-${crate.version}${lib.optionalString buildTests_ "-test"}"; version = crate.version; - depsBuildBuild = [ rust stdenv.cc cargo jq ]; + depsBuildBuild = [ pkgsBuildBuild.stdenv.cc ]; + nativeBuildInputs = [ rust stdenv.cc cargo jq ] ++ (crate.nativeBuildInputs or []) ++ nativeBuildInputs_; buildInputs = (crate.buildInputs or []) ++ buildInputs_; dependencies = map lib.getLib dependencies_; buildDependencies = map lib.getLib buildDependencies_; @@ -251,7 +254,7 @@ stdenv.mkDerivation (rec { depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies); hashedMetadata = builtins.hashString "sha256" (crateName + "-" + crateVersion + "___" + toString (mkRustcFeatureArgs crateFeatures) + - "___" + depsMetadata); + "___" + depsMetadata + "___" + rustAttrs.toRustTarget stdenv.hostPlatform); in lib.substring 0 10 hashedMetadata; build = crate.build or ""; @@ -279,7 +282,7 @@ stdenv.mkDerivation (rec { inherit crateName buildDependencies completeDeps completeBuildDeps crateDescription crateFeatures crateRenames libName build workspace_member release libPath crateVersion extraLinkFlags extraRustcOpts - crateAuthors crateHomepage verbose colors target_os; + crateAuthors crateHomepage verbose colors; }; buildPhase = buildCrate { inherit crateName dependencies @@ -301,6 +304,7 @@ stdenv.mkDerivation (rec { verbose = crate_.verbose or true; extraRustcOpts = []; features = []; + nativeBuildInputs = []; buildInputs = []; crateOverrides = defaultCrateOverrides; preUnpack = crate_.preUnpack or ""; diff --git a/pkgs/build-support/rust/build-rust-crate/test/default.nix b/pkgs/build-support/rust/build-rust-crate/test/default.nix index 112aa72fec7..65c8880b134 100644 --- a/pkgs/build-support/rust/build-rust-crate/test/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/test/default.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , buildRustCrate , callPackage , releaseTools @@ -10,13 +11,14 @@ }: let - mkCrate = args: let + mkCrate = buildRustCrate: args: let p = { crateName = "nixtestcrate"; version = "0.1.0"; authors = [ "Test <test@example.com>" ]; } // args; in buildRustCrate p; + mkHostCrate = mkCrate buildRustCrate; mkCargoToml = { name, crateVersion ? "0.1.0", path ? "Cargo.toml" }: @@ -68,15 +70,15 @@ let mkLib = name: mkFile name "pub fn test() -> i32 { return 23; }"; mkTest = crateArgs: let - crate = mkCrate (builtins.removeAttrs crateArgs ["expectedTestOutput"]); + crate = mkHostCrate (builtins.removeAttrs crateArgs ["expectedTestOutput"]); hasTests = crateArgs.buildTests or false; expectedTestOutputs = crateArgs.expectedTestOutputs or null; - binaries = map (v: ''"${v.name}"'') (crateArgs.crateBin or []); + binaries = map (v: lib.escapeShellArg v.name) (crateArgs.crateBin or []); isLib = crateArgs ? libName || crateArgs ? libPath; crateName = crateArgs.crateName or "nixtestcrate"; libName = crateArgs.libName or crateName; - libTestBinary = if !isLib then null else mkCrate { + libTestBinary = if !isLib then null else mkHostCrate { crateName = "run-test-${crateName}"; dependencies = [ crate ]; src = mkBinExtern "src/main.rs" libName; @@ -89,18 +91,27 @@ let runCommand "run-buildRustCrate-${crateName}-test" { nativeBuildInputs = [ crate ]; } (if !hasTests then '' - ${lib.concatStringsSep "\n" binaries} + ${lib.concatMapStringsSep "\n" (binary: + # Can't actually run the binary when cross-compiling + (lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) "type ") + binary + ) binaries} ${lib.optionalString isLib '' test -e ${crate}/lib/*.rlib || exit 1 - ${libTestBinary}/bin/run-test-${crateName} + ${lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) "test -x "} \ + ${libTestBinary}/bin/run-test-${crateName} ''} touch $out - '' else '' + '' else if stdenv.hostPlatform == stdenv.buildPlatform then '' for file in ${crate}/tests/*; do $file 2>&1 >> $out done set -e ${lib.concatMapStringsSep "\n" (o: "grep '${o}' $out || { echo 'output \"${o}\" not found in:'; cat $out; exit 23; }") expectedTestOutputs} + '' else '' + for file in ${crate}/tests/*; do + test -x "$file" + done + touch "$out" '' ); @@ -109,7 +120,7 @@ let `name` is used as part of the derivation name that performs the checking. - `crateArgs` is passed to `mkCrate` to build the crate with `buildRustCrate`. + `crateArgs` is passed to `mkHostCrate` to build the crate with `buildRustCrate`. `expectedFiles` contains a list of expected file paths in the output. E.g. `[ "./bin/my_binary" ]`. @@ -124,7 +135,7 @@ let assert (builtins.isList expectedFiles); let - crate = mkCrate (builtins.removeAttrs crateArgs ["expectedTestOutput"]); + crate = mkHostCrate (builtins.removeAttrs crateArgs ["expectedTestOutput"]); crateOutput = if output == null then crate else crate."${output}"; expectedFilesFile = writeTextFile { name = "expected-files-${name}"; @@ -135,12 +146,18 @@ let }; in runCommand "assert-outputs-${name}" { - } '' + } ('' local actualFiles=$(mktemp) cd "${crateOutput}" - find . -type f | sort >$actualFiles - diff -q ${expectedFilesFile} $actualFiles >/dev/null || { + find . -type f \ + | sort \ + '' + # sed out the hash because it differs per platform + + '' + | sed -E -e 's/-[0-9a-fA-F]{10}\.rlib/-HASH.rlib/g' \ + > "$actualFiles" + diff -q ${expectedFilesFile} "$actualFiles" > /dev/null || { echo -e "\033[0;1;31mERROR: Difference in expected output files in ${crateOutput} \033[0m" >&2 echo === Got: sed -e 's/^/ /' $actualFiles @@ -153,7 +170,7 @@ let exit 1 } touch $out - '' + '') ; in rec { @@ -188,17 +205,17 @@ let crateBinRename1 = { crateBin = [{ name = "my-binary-rename1"; }]; src = mkBinExtern "src/main.rs" "foo_renamed"; - dependencies = [ (mkCrate { crateName = "foo"; src = mkLib "src/lib.rs"; }) ]; + dependencies = [ (mkHostCrate { crateName = "foo"; src = mkLib "src/lib.rs"; }) ]; crateRenames = { "foo" = "foo_renamed"; }; }; crateBinRename2 = { crateBin = [{ name = "my-binary-rename2"; }]; src = mkBinExtern "src/main.rs" "foo_renamed"; - dependencies = [ (mkCrate { crateName = "foo"; libName = "foolib"; src = mkLib "src/lib.rs"; }) ]; + dependencies = [ (mkHostCrate { crateName = "foo"; libName = "foolib"; src = mkLib "src/lib.rs"; }) ]; crateRenames = { "foo" = "foo_renamed"; }; }; crateBinRenameMultiVersion = let - crateWithVersion = version: mkCrate { + crateWithVersion = version: mkHostCrate { crateName = "my_lib"; inherit version; src = mkFile "src/lib.rs" '' @@ -307,7 +324,7 @@ let fn main() {} ''; dependencies = [ - (mkCrate { + (mkHostCrate { crateName = "somerlib"; type = [ "rlib" ]; src = mkLib "src/lib.rs"; @@ -315,7 +332,7 @@ let ]; }; buildScriptDeps = let - depCrate = boolVal: mkCrate { + depCrate = buildRustCrate: boolVal: mkCrate buildRustCrate { crateName = "bar"; src = mkFile "src/lib.rs" '' pub const baz: bool = ${boolVal}; @@ -339,8 +356,8 @@ let '') ]; }; - buildDependencies = [ (depCrate "true") ]; - dependencies = [ (depCrate "false") ]; + buildDependencies = [ (depCrate buildPackages.buildRustCrate "true") ]; + dependencies = [ (depCrate buildRustCrate "false") ]; buildTests = true; expectedTestOutputs = [ "test baz_false ... ok" ]; }; @@ -373,7 +390,7 @@ let # Regression test for https://github.com/NixOS/nixpkgs/pull/88054 # Build script output should be rewritten as valid env vars. buildScriptIncludeDirDeps = let - depCrate = mkCrate { + depCrate = mkHostCrate { crateName = "bar"; src = symlinkJoin { name = "build-script-and-include-dir-bar"; @@ -460,7 +477,7 @@ let mkdir -p $out/lib # Note: On darwin (which defaults to clang) we have to add # `-undefined dynamic_lookup` as otherwise the compilation fails. - cc -shared \ + $CC -shared \ ${lib.optionalString stdenv.isDarwin "-undefined dynamic_lookup"} \ -o $out/lib/${name}${stdenv.hostPlatform.extensions.sharedLibrary} ${src} ''; @@ -519,6 +536,7 @@ let }; procMacroInPrelude = { procMacro = true; + edition = "2018"; src = symlinkJoin { name = "proc-macro-in-prelude"; paths = [ @@ -582,7 +600,7 @@ let }; expectedFiles = [ "./nix-support/propagated-build-inputs" - "./lib/libtest_lib-042a1fdbef.rlib" + "./lib/libtest_lib-HASH.rlib" "./lib/link" ]; }; @@ -599,7 +617,7 @@ let }; expectedFiles = [ "./nix-support/propagated-build-inputs" - "./lib/libtest_lib-042a1fdbef.rlib" + "./lib/libtest_lib-HASH.rlib" "./lib/link" ]; }; @@ -608,9 +626,11 @@ let pkg = brotliCrates.brotli_2_5_0 {}; in runCommand "run-brotli-test-cmd" { nativeBuildInputs = [ pkg ]; - } '' + } (if stdenv.hostPlatform == stdenv.buildPlatform then '' ${pkg}/bin/brotli -c ${pkg}/bin/brotli > /dev/null && touch $out - ''; + '' else '' + test -x '${pkg}/bin/brotli' && touch $out + ''); allocNoStdLibTest = let pkg = brotliCrates.alloc_no_stdlib_1_3_0 {}; in runCommand "run-alloc-no-stdlib-test-cmd" { diff --git a/pkgs/build-support/rust/default-crate-overrides.nix b/pkgs/build-support/rust/default-crate-overrides.nix index 1c4fe9daead..61cec2a6aba 100644 --- a/pkgs/build-support/rust/default-crate-overrides.nix +++ b/pkgs/build-support/rust/default-crate-overrides.nix @@ -1,6 +1,7 @@ -{ stdenv, pkgconfig, curl, darwin, libiconv, libgit2, libssh2, +{ lib, stdenv, pkg-config, curl, darwin, libiconv, libgit2, libssh2, openssl, sqlite, zlib, dbus, dbus-glib, gdk-pixbuf, cairo, python3, - libsodium, postgresql, gmp, foundationdb, ... }: + libsodium, postgresql, gmp, foundationdb, capnproto, nettle, clang, + llvmPackages, ... }: let inherit (darwin.apple_sdk.frameworks) CoreFoundation Security; @@ -10,24 +11,31 @@ in buildInputs = [ cairo ]; }; + capnp-rpc = attrs: { + nativeBuildInputs = [ capnproto ]; + }; + cargo = attrs: { buildInputs = [ openssl zlib curl ] - ++ stdenv.lib.optionals stdenv.isDarwin [ CoreFoundation Security libiconv ]; + ++ lib.optionals stdenv.isDarwin [ CoreFoundation Security libiconv ]; }; libz-sys = attrs: { - buildInputs = [ pkgconfig zlib ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ zlib ]; extraLinkFlags = ["-L${zlib.out}/lib"]; }; curl-sys = attrs: { - buildInputs = [ pkgconfig zlib curl ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ zlib curl ]; propagatedBuildInputs = [ curl zlib ]; extraLinkFlags = ["-L${zlib.out}/lib"]; }; dbus = attrs: { - buildInputs = [ pkgconfig dbus ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ dbus ]; }; foundationdb-sys = attrs: { @@ -62,19 +70,29 @@ in libgit2-sys = attrs: { LIBGIT2_SYS_USE_PKG_CONFIG = true; - buildInputs = [ pkgconfig openssl zlib libgit2 ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl zlib libgit2 ]; }; libsqlite3-sys = attrs: { - buildInputs = [ pkgconfig sqlite ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ sqlite ]; }; libssh2-sys = attrs: { - buildInputs = [ pkgconfig openssl zlib libssh2 ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl zlib libssh2 ]; }; libdbus-sys = attrs: { - buildInputs = [ pkgconfig dbus ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ dbus ]; + }; + + nettle-sys = attrs: { + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ nettle clang ]; + LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib"; }; openssl = attrs: { @@ -82,11 +100,13 @@ in }; openssl-sys = attrs: { - buildInputs = [ pkgconfig openssl ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl ]; }; pq-sys = attr: { - buildInputs = [ pkgconfig postgresql ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ postgresql ]; }; rink = attrs: { @@ -98,12 +118,43 @@ in propagatedBuildInputs = [ Security ]; }; + sequoia-openpgp = attrs: { + buildInputs = [ gmp ]; + }; + + sequoia-openpgp-ffi = attrs: { + buildInputs = [ gmp ]; + }; + + sequoia-ipc = attrs: { + buildInputs = [ gmp ]; + }; + + sequoia-guide = attrs: { + buildInputs = [ gmp ]; + }; + + sequoia-store = attrs: { + nativeBuildInputs = [ capnproto ]; + buildInputs = [ sqlite gmp ]; + }; + + sequoia-sq = attrs: { + buildInputs = [ sqlite gmp ]; + }; + + sequoia-tool = attrs: { + nativeBuildInputs = [ capnproto ]; + buildInputs = [ sqlite gmp ]; + }; + serde_derive = attrs: { - buildInputs = stdenv.lib.optional stdenv.isDarwin Security; + buildInputs = lib.optional stdenv.isDarwin Security; }; thrussh-libsodium = attrs: { - buildInputs = [ pkgconfig libsodium ]; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libsodium ]; }; xcb = attrs: { diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index c292b8ea4d4..be983af1c11 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -1,17 +1,35 @@ { stdenv +, lib , buildPackages , cacert -, cargo -, diffutils +, cargoBuildHook +, cargoCheckHook +, cargoInstallHook +, cargoSetupHook , fetchCargoTarball +, importCargoLock +, runCommandNoCC +, rustPlatform +, callPackage +, remarshal , git , rust , rustc +, libiconv , windows }: { name ? "${args.pname}-${args.version}" -, cargoSha256 ? "unset" + + # SRI hash +, cargoHash ? "" + + # Legacy hash +, cargoSha256 ? "" + + # Name for the vendored dependencies tarball +, cargoDepsName ? name + , src ? null , srcs ? null , unpackPhase ? null @@ -23,12 +41,15 @@ , nativeBuildInputs ? [] , cargoUpdateHook ? "" , cargoDepsHook ? "" -, cargoBuildFlags ? [] , buildType ? "release" , meta ? {} -, target ? null +, cargoLock ? null , cargoVendorDir ? null , checkType ? buildType +, depsExtraArgs ? {} + +# Toggles whether a custom sysroot is created when the target is a .json file. +, __internal_dontAddSysroot ? false # Needed to `pushd`/`popd` into a subdir of a tarball if this subdir # contains a Cargo.toml, but isn't part of a workspace (which is e.g. the @@ -37,59 +58,74 @@ , buildAndTestSubdir ? null , ... } @ args: -assert cargoVendorDir == null -> cargoSha256 != "unset"; +assert cargoVendorDir == null && cargoLock == null -> cargoSha256 == "" && cargoHash == "" + -> throw "cargoSha256, cargoHash, cargoVendorDir, or cargoLock must be set"; assert buildType == "release" || buildType == "debug"; let - cargoDeps = if cargoVendorDir == null - then fetchCargoTarball { - inherit name src srcs sourceRoot unpackPhase cargoUpdateHook; - patches = cargoPatches; - sha256 = cargoSha256; - } + cargoDeps = + if cargoVendorDir == null + then if cargoLock != null then importCargoLock cargoLock + else fetchCargoTarball ({ + inherit src srcs sourceRoot unpackPhase cargoUpdateHook; + name = cargoDepsName; + hash = cargoHash; + patches = cargoPatches; + sha256 = cargoSha256; + } // depsExtraArgs) else null; # If we have a cargoSha256 fixed-output derivation, validate it at build time # against the src fixed-output derivation to check consistency. - validateCargoDeps = cargoSha256 != "unset"; - - # Some cargo builds include build hooks that modify their own vendor - # dependencies. This copies the vendor directory into the build tree and makes - # it writable. If we're using a tarball, the unpackFile hook already handles - # this for us automatically. - setupVendorDir = if cargoVendorDir == null - then ('' - unpackFile "$cargoDeps" - cargoDepsCopy=$(stripHash $cargoDeps) - '') - else '' - cargoDepsCopy="$sourceRoot/${cargoVendorDir}" - ''; - - rustTarget = if target == null then rust.toRustTarget stdenv.hostPlatform else target; - - ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; - cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; - ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; - cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; - releaseDir = "target/${rustTarget}/${buildType}"; - tmpDir = "${releaseDir}-tmp"; - - # Specify the stdenv's `diff` by abspath to ensure that the user's build - # inputs do not cause us to find the wrong `diff`. - # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. - diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; + validateCargoDeps = !(cargoHash == "" && cargoSha256 == ""); + + target = rust.toRustTargetSpec stdenv.hostPlatform; + targetIsJSON = lib.hasSuffix ".json" target; + useSysroot = targetIsJSON && !__internal_dontAddSysroot; + + # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 + # the "${}" is needed to transform the path into a /nix/store path before baseNameOf + shortTarget = if targetIsJSON then + (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) + else target; + + sysroot = (callPackage ./sysroot {}) { + inherit target shortTarget; + RUSTFLAGS = args.RUSTFLAGS or ""; + originalCargoToml = src + /Cargo.toml; # profile info is later extracted + }; in -stdenv.mkDerivation (args // { - inherit cargoDeps; +# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`. +# See https://os.phil-opp.com/testing/ for more information. +assert useSysroot -> !(args.doCheck or true); + +stdenv.mkDerivation ((removeAttrs args [ "depsExtraArgs" "cargoLock" ]) // lib.optionalAttrs useSysroot { + RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or ""); +} // { + inherit buildAndTestSubdir cargoDeps; + + cargoBuildType = buildType; + + cargoCheckType = checkType; patchRegistryDeps = ./patch-registry-deps; - nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ]; - buildInputs = buildInputs ++ stdenv.lib.optional stdenv.hostPlatform.isMinGW windows.pthreads; + nativeBuildInputs = nativeBuildInputs ++ [ + cacert + git + cargoBuildHook + cargoCheckHook + cargoInstallHook + cargoSetupHook + rustc + ]; + + buildInputs = buildInputs + ++ lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ] + ++ lib.optionals stdenv.hostPlatform.isMinGW [ windows.pthreads ]; patches = cargoPatches ++ patches; @@ -99,146 +135,18 @@ stdenv.mkDerivation (args // { postUnpack = '' eval "$cargoDepsHook" - ${setupVendorDir} - - mkdir .cargo - config="$(pwd)/$cargoDepsCopy/.cargo/config"; - if [[ ! -e $config ]]; then - config=${./fetchcargo-default-config.toml}; - fi; - substitute $config .cargo/config \ - --subst-var-by vendor "$(pwd)/$cargoDepsCopy" - - cat >> .cargo/config <<'EOF' - [target."${rust.toRustTarget stdenv.buildPlatform}"] - "linker" = "${ccForBuild}" - ${stdenv.lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${rustTarget}"] - "linker" = "${ccForHost}" - ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 - stdenv.lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' - "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] - ''} - ''} - EOF - export RUST_LOG=${logLevel} '' + (args.postUnpack or ""); - # After unpacking and applying patches, check that the Cargo.lock matches our - # src package. Note that we do this after the patchPhase, because the - # patchPhase may create the Cargo.lock if upstream has not shipped one. - postPatch = (args.postPatch or "") + stdenv.lib.optionalString validateCargoDeps '' - cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock - srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock - - echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" - if ! ${diff} $srcLockfile $cargoDepsLockfile; then - - # If the diff failed, first double-check that the file exists, so we can - # give a friendlier error msg. - if ! [ -e $srcLockfile ]; then - echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" - echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." - exit 1 - fi - - if ! [ -e $cargoDepsLockfile ]; then - echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" - exit 1 - fi - - echo - echo "ERROR: cargoSha256 is out of date" - echo - echo "Cargo.lock is not the same in $cargoDepsCopy" - echo - echo "To fix the issue:" - echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' - echo "2. Build the derivation and wait it to fail with a hash mismatch" - echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" - echo - - exit 1 - fi - '' + '' - unset cargoDepsCopy - ''; - configurePhase = args.configurePhase or '' runHook preConfigure runHook postConfigure ''; - buildPhase = with builtins; args.buildPhase or '' - ${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"} - runHook preBuild - - ( - set -x - env \ - "CC_${rust.toRustTarget stdenv.buildPlatform}"="${ccForBuild}" \ - "CXX_${rust.toRustTarget stdenv.buildPlatform}"="${cxxForBuild}" \ - "CC_${rust.toRustTarget stdenv.hostPlatform}"="${ccForHost}" \ - "CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \ - cargo build \ - ${stdenv.lib.optionalString (buildType == "release") "--release"} \ - --target ${rustTarget} \ - --frozen ${concatStringsSep " " cargoBuildFlags} - ) - - runHook postBuild - - ${stdenv.lib.optionalString (buildAndTestSubdir != null) "popd"} - - # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in - # the pre/postBuild-hooks that need to be taken into account before gathering - # all binaries to install. - mkdir -p $tmpDir - cp -r $releaseDir/* $tmpDir/ - bins=$(find $tmpDir \ - -maxdepth 1 \ - -type f \ - -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \)) - ''; - - checkPhase = args.checkPhase or (let - argstr = "${stdenv.lib.optionalString (checkType == "release") "--release"} --target ${rustTarget} --frozen"; - in '' - ${stdenv.lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"} - runHook preCheck - echo "Running cargo test ${argstr} -- ''${checkFlags} ''${checkFlagsArray+''${checkFlagsArray[@]}}" - cargo test ${argstr} -- ''${checkFlags} ''${checkFlagsArray+"''${checkFlagsArray[@]}"} - runHook postCheck - ${stdenv.lib.optionalString (buildAndTestSubdir != null) "popd"} - ''); - doCheck = args.doCheck or true; strictDeps = true; - inherit releaseDir tmpDir; - - installPhase = args.installPhase or '' - runHook preInstall - - # rename the output dir to a architecture independent one - mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${tmpDir}$') - for target in "''${targets[@]}"; do - rm -rf "$target/../../${buildType}" - ln -srf "$target" "$target/../../" - done - mkdir -p $out/bin $out/lib - - xargs -r cp -t $out/bin <<< $bins - find $tmpDir \ - -maxdepth 1 \ - -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \ - -print0 | xargs -r -0 cp -t $out/lib - rmdir --ignore-fail-on-non-empty $out/lib $out/bin - runHook postInstall - ''; - passthru = { inherit cargoDeps; } // (args.passthru or {}); meta = { diff --git a/pkgs/build-support/rust/fetchCargoTarball.nix b/pkgs/build-support/rust/fetchCargoTarball.nix index dff5d99da9e..3b36554e707 100644 --- a/pkgs/build-support/rust/fetchCargoTarball.nix +++ b/pkgs/build-support/rust/fetchCargoTarball.nix @@ -1,4 +1,4 @@ -{ stdenv, cacert, git, cargo, python3 }: +{ lib, stdenv, cacert, git, cargo, python3 }: let cargo-vendor-normalise = stdenv.mkDerivation { name = "cargo-vendor-normalise"; src = ./cargo-vendor-normalise.py; @@ -21,12 +21,18 @@ in , src ? null , srcs ? [] , patches ? [] -, sourceRoot -, sha256 +, sourceRoot ? "" +, hash ? "" +, sha256 ? "" , cargoUpdateHook ? "" , ... } @ args: -stdenv.mkDerivation ({ + +let hash_ = + if hash != "" then { outputHashAlgo = null; outputHash = hash; } + else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; } + else throw "fetchCargoTarball requires a hash for ${name}"; +in stdenv.mkDerivation ({ name = "${name}-vendor.tar.gz"; nativeBuildInputs = [ cacert git cargo-vendor-normalise cargo ]; @@ -40,7 +46,7 @@ stdenv.mkDerivation ({ echo echo "ERROR: The Cargo.lock file doesn't exist" echo - echo "Cargo.lock is needed to make sure that cargoSha256 doesn't change" + echo "Cargo.lock is needed to make sure that cargoHash/cargoSha256 doesn't change" echo "when the registry is updated." echo @@ -72,10 +78,9 @@ stdenv.mkDerivation ({ -czf $out $name ''; - outputHashAlgo = "sha256"; - outputHash = sha256; + inherit (hash_) outputHashAlgo outputHash; - impureEnvVars = stdenv.lib.fetchers.proxyImpureEnvVars; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; } // (builtins.removeAttrs args [ "name" "sha256" "cargoUpdateHook" ])) diff --git a/pkgs/build-support/rust/fetchcrate.nix b/pkgs/build-support/rust/fetchcrate.nix index 95dfd38b12a..4e6c38b032c 100644 --- a/pkgs/build-support/rust/fetchcrate.nix +++ b/pkgs/build-support/rust/fetchcrate.nix @@ -1,10 +1,13 @@ { lib, fetchurl, unzip }: -{ crateName +{ crateName ? args.pname +, pname ? null , version , sha256 , ... } @ args: +assert pname == null || pname == crateName; + lib.overrideDerivation (fetchurl ({ name = "${crateName}-${version}.tar.gz"; @@ -30,6 +33,6 @@ lib.overrideDerivation (fetchurl ({ fi mv "$unpackDir/$fn" "$out" ''; -} // removeAttrs args [ "crateName" "version" ])) +} // removeAttrs args [ "crateName" "pname" "version" ])) # Hackety-hack: we actually need unzip hooks, too (x: {nativeBuildInputs = x.nativeBuildInputs++ [unzip];}) diff --git a/pkgs/build-support/rust/hooks/cargo-build-hook.sh b/pkgs/build-support/rust/hooks/cargo-build-hook.sh new file mode 100644 index 00000000000..c10120c5aa1 --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-build-hook.sh @@ -0,0 +1,41 @@ +declare -a cargoBuildFlags + +cargoBuildHook() { + echo "Executing cargoBuildHook" + + runHook preBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + pushd "${buildAndTestSubdir}" + fi + + if [ "${cargoBuildType}" != "debug" ]; then + cargoBuildProfileFlag="--${cargoBuildType}" + fi + + ( + set -x + env \ + "CC_@rustBuildPlatform@=@ccForBuild@" \ + "CXX_@rustBuildPlatform@=@cxxForBuild@" \ + "CC_@rustTargetPlatform@=@ccForHost@" \ + "CXX_@rustTargetPlatform@=@cxxForHost@" \ + cargo build -j $NIX_BUILD_CORES \ + --target @rustTargetPlatformSpec@ \ + --frozen \ + ${cargoBuildProfileFlag} \ + ${cargoBuildFlags} + ) + + if [ ! -z "${buildAndTestSubdir-}" ]; then + popd + fi + + runHook postBuild + + echo "Finished cargoBuildHook" +} + +if [ -z "${dontCargoBuild-}" ] && [ -z "${buildPhase-}" ]; then + buildPhase=cargoBuildHook +fi diff --git a/pkgs/build-support/rust/hooks/cargo-check-hook.sh b/pkgs/build-support/rust/hooks/cargo-check-hook.sh new file mode 100644 index 00000000000..f0339afb38f --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-check-hook.sh @@ -0,0 +1,46 @@ +declare -a checkFlags +declare -a cargoTestFlags + +cargoCheckHook() { + echo "Executing cargoCheckHook" + + runHook preCheck + + if [[ -n "${buildAndTestSubdir-}" ]]; then + pushd "${buildAndTestSubdir}" + fi + + if [[ -z ${dontUseCargoParallelTests-} ]]; then + threads=$NIX_BUILD_CORES + else + threads=1 + fi + + if [ "${cargoBuildType}" != "debug" ]; then + cargoBuildProfileFlag="--${cargoBuildType}" + fi + + argstr="${cargoBuildProfileFlag} --target @rustTargetPlatformSpec@ --frozen ${cargoTestFlags}"; + + ( + set -x + cargo test \ + -j $NIX_BUILD_CORES \ + ${argstr} -- \ + --test-threads=${threads} \ + ${checkFlags} \ + ${checkFlagsArray+"${checkFlagsArray[@]}"} + ) + + if [[ -n "${buildAndTestSubdir-}" ]]; then + popd + fi + + echo "Finished cargoCheckHook" + + runHook postCheck +} + +if [ -z "${dontCargoCheck-}" ] && [ -z "${checkPhase-}" ]; then + checkPhase=cargoCheckHook +fi diff --git a/pkgs/build-support/rust/hooks/cargo-install-hook.sh b/pkgs/build-support/rust/hooks/cargo-install-hook.sh new file mode 100644 index 00000000000..69ce7266936 --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-install-hook.sh @@ -0,0 +1,49 @@ +cargoInstallPostBuildHook() { + echo "Executing cargoInstallPostBuildHook" + + releaseDir=target/@shortTarget@/$cargoBuildType + tmpDir="${releaseDir}-tmp"; + + mkdir -p $tmpDir + cp -r ${releaseDir}/* $tmpDir/ + bins=$(find $tmpDir \ + -maxdepth 1 \ + -type f \ + -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \)) + + echo "Finished cargoInstallPostBuildHook" +} + +cargoInstallHook() { + echo "Executing cargoInstallHook" + + runHook preInstall + + # rename the output dir to a architecture independent one + + releaseDir=target/@shortTarget@/$cargoBuildType + tmpDir="${releaseDir}-tmp"; + + mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep "${tmpDir}$") + for target in "${targets[@]}"; do + rm -rf "$target/../../${cargoBuildType}" + ln -srf "$target" "$target/../../" + done + mkdir -p $out/bin $out/lib + + xargs -r cp -t $out/bin <<< $bins + find $tmpDir \ + -maxdepth 1 \ + -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \ + -print0 | xargs -r -0 cp -t $out/lib + rmdir --ignore-fail-on-non-empty $out/lib $out/bin + runHook postInstall + + echo "Finished cargoInstallHook" +} + + +if [ -z "${dontCargoInstall-}" ] && [ -z "${installPhase-}" ]; then + installPhase=cargoInstallHook + postBuildHooks+=(cargoInstallPostBuildHook) +fi diff --git a/pkgs/build-support/rust/hooks/cargo-setup-hook.sh b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh new file mode 100644 index 00000000000..842e66b5170 --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh @@ -0,0 +1,86 @@ +cargoSetupPostUnpackHook() { + echo "Executing cargoSetupPostUnpackHook" + + # Some cargo builds include build hooks that modify their own vendor + # dependencies. This copies the vendor directory into the build tree and makes + # it writable. If we're using a tarball, the unpackFile hook already handles + # this for us automatically. + if [ -z $cargoVendorDir ]; then + unpackFile "$cargoDeps" + export cargoDepsCopy=$(stripHash $cargoDeps) + else + cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}" + fi + + if [ ! -d .cargo ]; then + mkdir .cargo + fi + + config="$(pwd)/$cargoDepsCopy/.cargo/config"; + if [[ ! -e $config ]]; then + config=@defaultConfig@ + fi; + + tmp_config=$(mktemp) + substitute $config $tmp_config \ + --subst-var-by vendor "$(pwd)/$cargoDepsCopy" + cat ${tmp_config} >> .cargo/config + + cat >> .cargo/config <<'EOF' + @rustTarget@ +EOF + + echo "Finished cargoSetupPostUnpackHook" +} + +# After unpacking and applying patches, check that the Cargo.lock matches our +# src package. Note that we do this after the patchPhase, because the +# patchPhase may create the Cargo.lock if upstream has not shipped one. +cargoSetupPostPatchHook() { + echo "Executing cargoSetupPostPatchHook" + + cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock" + srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock" + + echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" + if ! @diff@ $srcLockfile $cargoDepsLockfile; then + + # If the diff failed, first double-check that the file exists, so we can + # give a friendlier error msg. + if ! [ -e $srcLockfile ]; then + echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" + echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." + exit 1 + fi + + if ! [ -e $cargoDepsLockfile ]; then + echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" + exit 1 + fi + + echo + echo "ERROR: cargoSha256 is out of date" + echo + echo "Cargo.lock is not the same in $cargoDepsCopy" + echo + echo "To fix the issue:" + echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" + echo + + exit 1 + fi + + unset cargoDepsCopy + + echo "Finished cargoSetupPostPatchHook" +} + +if [ -z "${dontCargoSetupPostUnpack-}" ]; then + postUnpackHooks+=(cargoSetupPostUnpackHook) +fi + +if [ -z ${cargoVendorDir-} ]; then + postPatchHooks+=(cargoSetupPostPatchHook) +fi diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix new file mode 100644 index 00000000000..d86c9ebaed8 --- /dev/null +++ b/pkgs/build-support/rust/hooks/default.nix @@ -0,0 +1,95 @@ +{ buildPackages +, callPackage +, cargo +, diffutils +, lib +, makeSetupHook +, maturin +, rust +, rustc +, stdenv +, target ? rust.toRustTargetSpec stdenv.hostPlatform +}: + +let + targetIsJSON = lib.hasSuffix ".json" target; + + # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 + # the "${}" is needed to transform the path into a /nix/store path before baseNameOf + shortTarget = if targetIsJSON then + (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) + else target; + ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; + cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; + ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; + cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; + rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform; + rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform; + rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; +in { + cargoBuildHook = callPackage ({ }: + makeSetupHook { + name = "cargo-build-hook.sh"; + deps = [ cargo ]; + substitutions = { + inherit ccForBuild ccForHost cxxForBuild cxxForHost + rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + }; + } ./cargo-build-hook.sh) {}; + + cargoCheckHook = callPackage ({ }: + makeSetupHook { + name = "cargo-check-hook.sh"; + deps = [ cargo ]; + substitutions = { + inherit rustTargetPlatformSpec; + }; + } ./cargo-check-hook.sh) {}; + + cargoInstallHook = callPackage ({ }: + makeSetupHook { + name = "cargo-install-hook.sh"; + deps = [ ]; + substitutions = { + inherit shortTarget; + }; + } ./cargo-install-hook.sh) {}; + + cargoSetupHook = callPackage ({ }: + makeSetupHook { + name = "cargo-setup-hook.sh"; + deps = [ ]; + substitutions = { + defaultConfig = ../fetchcargo-default-config.toml; + + # Specify the stdenv's `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong `diff`. + # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. + diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; + + # Target platform + rustTarget = '' + [target."${rust.toRustTarget stdenv.buildPlatform}"] + "linker" = "${ccForBuild}" + ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' + [target."${shortTarget}"] + "linker" = "${ccForHost}" + ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 + lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' + "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] + ''} + ''} + ''; + }; + } ./cargo-setup-hook.sh) {}; + + maturinBuildHook = callPackage ({ }: + makeSetupHook { + name = "maturin-build-hook.sh"; + deps = [ cargo maturin rustc ]; + substitutions = { + inherit ccForBuild ccForHost cxxForBuild cxxForHost + rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + }; + } ./maturin-build-hook.sh) {}; +} diff --git a/pkgs/build-support/rust/hooks/maturin-build-hook.sh b/pkgs/build-support/rust/hooks/maturin-build-hook.sh new file mode 100644 index 00000000000..7e2599d9224 --- /dev/null +++ b/pkgs/build-support/rust/hooks/maturin-build-hook.sh @@ -0,0 +1,39 @@ +maturinBuildHook() { + echo "Executing maturinBuildHook" + + runHook preBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + pushd "${buildAndTestSubdir}" + fi + + ( + set -x + env \ + "CC_@rustBuildPlatform@=@ccForBuild@" \ + "CXX_@rustBuildPlatform@=@cxxForBuild@" \ + "CC_@rustTargetPlatform@=@ccForHost@" \ + "CXX_@rustTargetPlatform@=@cxxForHost@" \ + maturin build \ + --cargo-extra-args="-j $NIX_BUILD_CORES --frozen" \ + --target @rustTargetPlatformSpec@ \ + --manylinux off \ + --strip \ + --release \ + ${maturinBuildFlags-} + ) + + runHook postBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + popd + fi + + # Move the wheel to dist/ so that regular Python tooling can find it. + mkdir -p dist + mv target/wheels/*.whl dist/ + + echo "Finished maturinBuildHook" +} + +buildPhase=maturinBuildHook diff --git a/pkgs/build-support/rust/import-cargo-lock.nix b/pkgs/build-support/rust/import-cargo-lock.nix new file mode 100644 index 00000000000..83f4e0df4f2 --- /dev/null +++ b/pkgs/build-support/rust/import-cargo-lock.nix @@ -0,0 +1,175 @@ +{ fetchgit, fetchurl, lib, runCommand, cargo, jq }: + +{ + # Cargo lock file + lockFile + + # Hashes for git dependencies. +, outputHashes ? {} +}: + +let + # Parse a git source into different components. + parseGit = src: + let + parts = builtins.match ''git\+([^?]+)(\?rev=(.*))?#(.*)?'' src; + rev = builtins.elemAt parts 2; + in + if parts == null then null + else { + url = builtins.elemAt parts 0; + sha = builtins.elemAt parts 3; + } // lib.optionalAttrs (rev != null) { inherit rev; }; + + packages = (builtins.fromTOML (builtins.readFile lockFile)).package; + + # There is no source attribute for the source package itself. But + # since we do not want to vendor the source package anyway, we can + # safely skip it. + depPackages = (builtins.filter (p: p ? "source") packages); + + # Create dependent crates from packages. + # + # Force evaluation of the git SHA -> hash mapping, so that an error is + # thrown if there are stale hashes. We cannot rely on gitShaOutputHash + # being evaluated otherwise, since there could be no git dependencies. + depCrates = builtins.deepSeq (gitShaOutputHash) (builtins.map mkCrate depPackages); + + # Map package name + version to git commit SHA for packages with a git source. + namesGitShas = builtins.listToAttrs ( + builtins.map nameGitSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages) + ); + + nameGitSha = pkg: let gitParts = parseGit pkg.source; in { + name = "${pkg.name}-${pkg.version}"; + value = gitParts.sha; + }; + + # Convert the attrset provided through the `outputHashes` argument to a + # a mapping from git commit SHA -> output hash. + # + # There may be multiple different packages with different names + # originating from the same git repository (typically a Cargo + # workspace). By using the git commit SHA as a universal identifier, + # the user does not have to specify the output hash for every package + # individually. + gitShaOutputHash = lib.mapAttrs' (nameVer: hash: + let + unusedHash = throw "A hash was specified for ${nameVer}, but there is no corresponding git dependency."; + rev = namesGitShas.${nameVer} or unusedHash; in { + name = rev; + value = hash; + }) outputHashes; + + # We can't use the existing fetchCrate function, since it uses a + # recursive hash of the unpacked crate. + fetchCrate = pkg: + assert lib.assertMsg (pkg ? checksum) '' + Package ${pkg.name} does not have a checksum. + Please note that the Cargo.lock format where checksums used to be listed + under [metadata] is not supported. + If that is the case, running `cargo update` with a recent toolchain will + automatically update the format along with the crate's depenendencies. + ''; + fetchurl { + name = "crate-${pkg.name}-${pkg.version}.tar.gz"; + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = pkg.checksum; + }; + + # Fetch and unpack a crate. + mkCrate = pkg: + let + gitParts = parseGit pkg.source; + in + if pkg.source == "registry+https://github.com/rust-lang/crates.io-index" then + let + crateTarball = fetchCrate pkg; + in runCommand "${pkg.name}-${pkg.version}" {} '' + mkdir $out + tar xf "${crateTarball}" -C $out --strip-components=1 + + # Cargo is happy with largely empty metadata. + printf '{"files":{},"package":"${pkg.checksum}"}' > "$out/.cargo-checksum.json" + '' + else if gitParts != null then + let + missingHash = throw '' + No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version}. You can add + a hash through the `outputHashes` argument of `importCargoLock`: + + outputHashes = { + "${pkg.name}-${pkg.version}" = "<hash>"; + }; + + If you use `buildRustPackage`, you can add this attribute to the `cargoLock` + attribute set. + ''; + sha256 = gitShaOutputHash.${gitParts.sha} or missingHash; + tree = fetchgit { + inherit sha256; + inherit (gitParts) url; + rev = gitParts.sha; # The commit SHA is always available. + }; + in runCommand "${pkg.name}-${pkg.version}" {} '' + tree=${tree} + if grep --quiet '\[workspace\]' "$tree/Cargo.toml"; then + # If the target package is in a workspace, find the crate path + # using `cargo metadata`. + crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \ + ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path') + + if [[ ! -z $crateCargoTOML ]]; then + tree=$(dirname $crateCargoTOML) + else + >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the Cargo workspace in: $tree" + exit 1 + fi + fi + + cp -prvd "$tree/" $out + chmod u+w $out + + # Cargo is happy with empty metadata. + printf '{"files":{},"package":null}' > "$out/.cargo-checksum.json" + + # Set up configuration for the vendor directory. + cat > $out/.cargo-config <<EOF + [source."${gitParts.url}"] + git = "${gitParts.url}" + ${lib.optionalString (gitParts ? rev) "rev = \"${gitParts.rev}\""} + replace-with = "vendored-sources" + EOF + '' + else throw "Cannot handle crate source: ${pkg.source}"; + + vendorDir = runCommand "cargo-vendor-dir" {} '' + mkdir -p $out/.cargo + + ln -s ${lockFile} $out/Cargo.lock + + cat > $out/.cargo/config <<EOF + [source.crates-io] + replace-with = "vendored-sources" + + [source.vendored-sources] + directory = "cargo-vendor-dir" + EOF + + declare -A keysSeen + + for crate in ${toString depCrates}; do + # Link the crate directory, removing the output path hash from the destination. + ln -s "$crate" $out/$(basename "$crate" | cut -c 34-) + + if [ -e "$crate/.cargo-config" ]; then + key=$(sed 's/\[source\."\(.*\)"\]/\1/; t; d' < "$crate/.cargo-config") + if [[ -z ''${keysSeen[$key]} ]]; then + keysSeen[$key]=1 + cat "$crate/.cargo-config" >> $out/.cargo/config + fi + fi + done + ''; +in + vendorDir diff --git a/pkgs/build-support/rust/sysroot/Cargo.lock b/pkgs/build-support/rust/sysroot/Cargo.lock new file mode 100644 index 00000000000..61fcef61744 --- /dev/null +++ b/pkgs/build-support/rust/sysroot/Cargo.lock @@ -0,0 +1,29 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "alloc" +version = "0.0.0" +dependencies = [ + "compiler_builtins", + "core", +] + +[[package]] +name = "compiler_builtins" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369" +dependencies = [ + "rustc-std-workspace-core", +] + +[[package]] +name = "core" +version = "0.0.0" + +[[package]] +name = "rustc-std-workspace-core" +version = "1.99.0" +dependencies = [ + "core", +] diff --git a/pkgs/build-support/rust/sysroot/cargo.py b/pkgs/build-support/rust/sysroot/cargo.py new file mode 100644 index 00000000000..09f6fba6d1c --- /dev/null +++ b/pkgs/build-support/rust/sysroot/cargo.py @@ -0,0 +1,45 @@ +import os +import toml + +rust_src = os.environ['RUSTC_SRC'] +orig_cargo = os.environ['ORIG_CARGO'] if 'ORIG_CARGO' in os.environ else None + +base = { + 'package': { + 'name': 'alloc', + 'version': '0.0.0', + 'authors': ['The Rust Project Developers'], + 'edition': '2018', + }, + 'dependencies': { + 'compiler_builtins': { + 'version': '0.1.0', + 'features': ['rustc-dep-of-std', 'mem'], + }, + 'core': { + 'path': os.path.join(rust_src, 'libcore'), + }, + }, + 'lib': { + 'name': 'alloc', + 'path': os.path.join(rust_src, 'liballoc/lib.rs'), + }, + 'patch': { + 'crates-io': { + 'rustc-std-workspace-core': { + 'path': os.path.join(rust_src, 'tools/rustc-std-workspace-core'), + }, + }, + }, +} + +if orig_cargo is not None: + with open(orig_cargo, 'r') as f: + src = toml.loads(f.read()) + if 'profile' in src: + base['profile'] = src['profile'] + +out = toml.dumps(base) + +with open('Cargo.toml', 'x') as f: + f.write(out) diff --git a/pkgs/build-support/rust/sysroot/default.nix b/pkgs/build-support/rust/sysroot/default.nix new file mode 100644 index 00000000000..4db7cf0dc39 --- /dev/null +++ b/pkgs/build-support/rust/sysroot/default.nix @@ -0,0 +1,41 @@ +{ stdenv, rust, rustPlatform, buildPackages }: + +{ shortTarget, originalCargoToml, target, RUSTFLAGS }: + +let + cargoSrc = stdenv.mkDerivation { + name = "cargo-src"; + preferLocalBuild = true; + phases = [ "installPhase" ]; + installPhase = '' + RUSTC_SRC=${rustPlatform.rustcSrc.override { minimalContent = false; }} ORIG_CARGO=${originalCargoToml} \ + ${buildPackages.python3.withPackages (ps: with ps; [ toml ])}/bin/python3 ${./cargo.py} + mkdir -p $out + cp Cargo.toml $out/Cargo.toml + cp ${./Cargo.lock} $out/Cargo.lock + ''; + }; +in rustPlatform.buildRustPackage { + inherit target RUSTFLAGS; + + name = "custom-sysroot"; + src = cargoSrc; + + RUSTC_BOOTSTRAP = 1; + __internal_dontAddSysroot = true; + cargoSha256 = "0y6dqfhsgk00y3fv5bnjzk0s7i30nwqc1rp0xlrk83hkh80x81mw"; + + doCheck = false; + + installPhase = '' + export LIBS_DIR=$out/lib/rustlib/${shortTarget}/lib + mkdir -p $LIBS_DIR + for f in target/${shortTarget}/release/deps/*.{rlib,rmeta}; do + cp $f $LIBS_DIR + done + + export RUST_SYSROOT=$(rustc --print=sysroot) + host=${rust.toRustTarget stdenv.buildPlatform} + cp -r $RUST_SYSROOT/lib/rustlib/$host $out + ''; +} diff --git a/pkgs/build-support/rust/sysroot/update-lockfile.sh b/pkgs/build-support/rust/sysroot/update-lockfile.sh new file mode 100755 index 00000000000..83d29832384 --- /dev/null +++ b/pkgs/build-support/rust/sysroot/update-lockfile.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p python3 python3.pkgs.toml cargo + +set -e + +HERE=$(dirname "${BASH_SOURCE[0]}") +NIXPKGS_ROOT="$HERE/../../../.." + +# https://unix.stackexchange.com/a/84980/390173 +tempdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-lockfile') + +cd "$tempdir" +nix-build -E "with import (/. + \"${NIXPKGS_ROOT}\") {}; pkgs.rustPlatform.rustcSrc.override { minimalContent = false; }" +RUSTC_SRC="$(pwd)/result" python3 "$HERE/cargo.py" +RUSTC_BOOTSTRAP=1 cargo build || echo "Build failure is expected. All that's needed is the lockfile." + +cp Cargo.lock "$HERE" + +rm -rf "$tempdir" + + diff --git a/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.lock b/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.lock new file mode 100644 index 00000000000..fd1b5e42ad3 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.lock @@ -0,0 +1,83 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "basic" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" diff --git a/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.toml b/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.toml new file mode 100644 index 00000000000..f555bb0de62 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/basic/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "basic" +version = "0.1.0" +authors = ["Daniël de Kok <me@danieldk.eu>"] +edition = "2018" + +[dependencies] +rand = "0.8" diff --git a/pkgs/build-support/rust/test/import-cargo-lock/basic/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/basic/default.nix new file mode 100644 index 00000000000..d595b58109a --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/basic/default.nix @@ -0,0 +1,18 @@ +{ rustPlatform }: + +rustPlatform.buildRustPackage { + pname = "basic"; + version = "0.1.0"; + + src = ./.; + + cargoLock = { + lockFile = ./Cargo.lock; + }; + + doInstallCheck = true; + + installCheckPhase = '' + $out/bin/basic + ''; +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/basic/src/main.rs b/pkgs/build-support/rust/test/import-cargo-lock/basic/src/main.rs new file mode 100644 index 00000000000..50b4ed799e4 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/basic/src/main.rs @@ -0,0 +1,9 @@ +use rand::Rng; + +fn main() { + let mut rng = rand::thread_rng(); + + // Always draw zero :). + let roll: u8 = rng.gen_range(0..1); + assert_eq!(roll, 0); +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/default.nix new file mode 100644 index 00000000000..2dd525a8ac3 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/default.nix @@ -0,0 +1,8 @@ +{ callPackage }: + +{ + basic = callPackage ./basic { }; + gitDependency = callPackage ./git-dependency { }; + gitDependencyNoRev = callPackage ./git-dependency-no-rev { }; + maturin = callPackage ./maturin { }; +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.lock b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.lock new file mode 100644 index 00000000000..54b9c7c5739 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.lock @@ -0,0 +1,79 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "git-dependency-no-rev" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "rand" +version = "0.8.3" +source = "git+https://github.com/rust-random/rand.git#f0e01ee0a7257753cc51b291f62666f4765923ef" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git#f0e01ee0a7257753cc51b291f62666f4765923ef" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "git+https://github.com/rust-random/rand.git#f0e01ee0a7257753cc51b291f62666f4765923ef" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git#f0e01ee0a7257753cc51b291f62666f4765923ef" +dependencies = [ + "rand_core", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.toml b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.toml new file mode 100644 index 00000000000..770dfb86f52 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "git-dependency-no-rev" +version = "0.1.0" +authors = ["Daniël de Kok <me@danieldk.eu>"] +edition = "2018" + +[dependencies] +rand = { git = "https://github.com/rust-random/rand.git" } diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/default.nix new file mode 100644 index 00000000000..fc36edc4077 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/default.nix @@ -0,0 +1,21 @@ +{ rustPlatform }: + +rustPlatform.buildRustPackage { + pname = "git-dependency-no-rev"; + version = "0.1.0"; + + src = ./.; + + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "rand-0.8.3" = "0ya2hia3cn31qa8894s3av2s8j5bjwb6yq92k0jsnlx7jid0jwqa"; + }; + }; + + doInstallCheck = true; + + installCheckPhase = '' + $out/bin/git-dependency-no-rev + ''; +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/src/main.rs b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/src/main.rs new file mode 100644 index 00000000000..50b4ed799e4 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-no-rev/src/main.rs @@ -0,0 +1,9 @@ +use rand::Rng; + +fn main() { + let mut rng = rand::thread_rng(); + + // Always draw zero :). + let roll: u8 = rng.gen_range(0..1); + assert_eq!(roll, 0); +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.lock b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.lock new file mode 100644 index 00000000000..50600ef4caa --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.lock @@ -0,0 +1,79 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "git-dependency" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "rand" +version = "0.8.3" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.1" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.toml b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.toml new file mode 100644 index 00000000000..11ee8b1763e --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "git-dependency" +version = "0.1.0" +authors = ["Daniël de Kok <me@danieldk.eu>"] +edition = "2018" + +[dependencies] +rand = { git = "https://github.com/rust-random/rand.git", rev = "0.8.3" } diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/default.nix new file mode 100644 index 00000000000..17276c5f5c3 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/default.nix @@ -0,0 +1,21 @@ +{ rustPlatform }: + +rustPlatform.buildRustPackage { + pname = "git-dependency"; + version = "0.1.0"; + + src = ./.; + + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "rand-0.8.3" = "0l3p174bpwia61vcvxz5mw65a13ri3wy94z04xrnyy5lzciykz4f"; + }; + }; + + doInstallCheck = true; + + installCheckPhase = '' + $out/bin/git-dependency + ''; +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/src/main.rs b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/src/main.rs new file mode 100644 index 00000000000..50b4ed799e4 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency/src/main.rs @@ -0,0 +1,9 @@ +use rand::Rng; + +fn main() { + let mut rng = rand::thread_rng(); + + // Always draw zero :). + let roll: u8 = rng.gen_range(0..1); + assert_eq!(roll, 0); +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/maturin/Cargo.lock b/pkgs/build-support/rust/test/import-cargo-lock/maturin/Cargo.lock new file mode 100644 index 00000000000..5e698d4ff73 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/maturin/Cargo.lock @@ -0,0 +1,682 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "assert_approx_eq" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c07dab4369547dbe5114677b33fbbf724971019f3818172d59a97a61c774ffd" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "const_fn" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "ctor" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "ghost" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "indoc" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +dependencies = [ + "indoc-impl", + "proc-macro-hack", +] + +[[package]] +name = "indoc-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", + "unindent", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "inventory" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0f7efb804ec95e33db9ad49e4252f049e37e8b0a4652e3cd61f7999f2eff7f" +dependencies = [ + "ctor", + "ghost", + "inventory-impl", +] + +[[package]] +name = "inventory-impl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c094e94816723ab936484666968f5b58060492e880f3c8d00489a1e244fa51" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num-bigint" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d632c0c558b87dbabbe6a82f3b4ae03720d0646ac5b7b4dae89394be5f2c5" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "proptest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e6c80c1139113c28ee4670dc50cc42915228b51f56a9e407f0ec60f966646f" +dependencies = [ + "bitflags", + "byteorder", + "lazy_static", + "num-traits", + "quick-error", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", +] + +[[package]] +name = "pyo3" +version = "0.13.2" +dependencies = [ + "assert_approx_eq", + "cfg-if", + "ctor", + "hashbrown", + "indoc", + "inventory", + "libc", + "num-bigint", + "num-complex", + "parking_lot", + "paste", + "proptest", + "pyo3", + "pyo3-macros", + "rustversion", + "serde", + "serde_json", + "trybuild", + "unindent", +] + +[[package]] +name = "pyo3-macros" +version = "0.13.2" +dependencies = [ + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.13.2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "rustapi-module" +version = "0.1.0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "rustversion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "trybuild" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99471a206425fba51842a9186315f32d91c56eadc21ea4c21f847b59cf778f8b" +dependencies = [ + "glob", + "lazy_static", + "serde", + "serde_json", + "termcolor", + "toml", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "word-count" +version = "0.1.0" +dependencies = [ + "pyo3", + "rayon", +] diff --git a/pkgs/build-support/rust/test/import-cargo-lock/maturin/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/maturin/default.nix new file mode 100644 index 00000000000..af0de596b38 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/maturin/default.nix @@ -0,0 +1,43 @@ +{ lib +, fetchFromGitHub +, python3 +, rustPlatform +}: + +python3.pkgs.buildPythonPackage rec { + pname = "word-count"; + version = "0.13.2"; + + format = "pyproject"; + + src = fetchFromGitHub { + owner = "PyO3"; + repo = "pyo3"; + rev = "v${version}"; + hash = "sha256-NOMrrfo8WjlPhtGxWUOPJS/UDDdbLQRCXR++Zd6JmIA="; + }; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ./Cargo.lock; + }; + + postPatch = '' + cp ${./Cargo.lock} Cargo.lock + ''; + + buildAndTestSubdir = "examples/word-count"; + + nativeBuildInputs = with rustPlatform; [ + cargoSetupHook + maturinBuildHook + ]; + + pythonImportsCheck = [ "word_count" ]; + + meta = with lib; { + description = "PyO3 word count example"; + homepage = "https://github.com/PyO3/pyo3"; + license = licenses.asl20; + maintainers = [ maintainers.danieldk ]; + }; +} diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 4f7c0c14304..511371931de 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -1,9 +1,20 @@ +#!/usr/bin/env bash + declare -a autoPatchelfLibs +declare -Ag autoPatchelfFailedDeps gatherLibraries() { autoPatchelfLibs+=("$1/lib") } +# wrapper around patchelf to raise proper error messages +# containing the tried file name and command +runPatchelf() { + patchelf "$@" || (echo "Command failed: patchelf $*" && exit 1) +} + +# shellcheck disable=SC2154 +# (targetOffset is referenced but not assigned.) addEnvHooks "$targetOffset" gatherLibraries isExecutable() { @@ -23,14 +34,19 @@ isExecutable() { # We cache dependencies so that we don't need to search through all of them on # every consecutive call to findDependency. -declare -a cachedDependencies +declare -Ag autoPatchelfCachedDepsAssoc +declare -ag autoPatchelfCachedDeps + addToDepCache() { - local existing - for existing in "${cachedDependencies[@]}"; do - if [ "$existing" = "$1" ]; then return; fi - done - cachedDependencies+=("$1") + if [[ ${autoPatchelfCachedDepsAssoc[$1]+f} ]]; then return; fi + + # store deps in an assoc. array for efficient lookups + # otherwise findDependency would have quadratic complexity + autoPatchelfCachedDepsAssoc["$1"]="" + + # also store deps in normal array to maintain their order + autoPatchelfCachedDeps+=("$1") } declare -gi depCacheInitialised=0 @@ -43,9 +59,8 @@ getDepsFromSo() { populateCacheWithRecursiveDeps() { local so found foundso - for so in "${cachedDependencies[@]}"; do + for so in "${autoPatchelfCachedDeps[@]}"; do for found in $(getDepsFromSo "$so"); do - local libdir="${found%/*}" local base="${found##*/}" local soname="${base%.so*}" for foundso in "${found%/*}/$soname".so*; do @@ -76,7 +91,7 @@ findDependency() { depCacheInitialised=1 fi - for dep in "${cachedDependencies[@]}"; do + for dep in "${autoPatchelfCachedDeps[@]}"; do if [ "$filename" = "${dep##*/}" ]; then if [ "$(getSoArch "$dep")" = "$arch" ]; then foundDependency="$dep" @@ -101,9 +116,12 @@ findDependency() { autoPatchelfFile() { local dep rpath="" toPatch="$1" - local interpreter="$(< "$NIX_CC/nix-support/dynamic-linker")" + local interpreter + interpreter="$(< "$NIX_CC/nix-support/dynamic-linker")" if isExecutable "$toPatch"; then - patchelf --set-interpreter "$interpreter" "$toPatch" + runPatchelf --set-interpreter "$interpreter" "$toPatch" + # shellcheck disable=SC2154 + # (runtimeDependencies is referenced but not assigned.) if [ -n "$runtimeDependencies" ]; then for dep in $runtimeDependencies; do rpath="$rpath${rpath:+:}$dep/lib" @@ -115,17 +133,19 @@ autoPatchelfFile() { # We're going to find all dependencies based on ldd output, so we need to # clear the RPATH first. - patchelf --remove-rpath "$toPatch" + runPatchelf --remove-rpath "$toPatch" - local missing="$( + # If the file is not a dynamic executable, ldd/sed will fail, + # in which case we return, since there is nothing left to do. + local missing + missing="$( ldd "$toPatch" 2> /dev/null | \ sed -n -e 's/^[\t ]*\([^ ]\+\) => not found.*/\1/p' - )" + )" || return 0 # This ensures that we get the output of all missing dependencies instead # of failing at the first one, because it's more useful when working on a # new package where you don't yet know its dependencies. - local -i depNotFound=0 for dep in $missing; do echo -n " $dep -> " >&2 @@ -134,18 +154,13 @@ autoPatchelfFile() { echo "found: $foundDependency" >&2 else echo "not found!" >&2 - depNotFound=1 + autoPatchelfFailedDeps["$dep"]="$toPatch" fi done - # This makes sure the builder fails if we didn't find a dependency, because - # the stdenv setup script is run with set -e. The actual error is emitted - # earlier in the previous loop. - [ $depNotFound -eq 0 -o -n "$autoPatchelfIgnoreMissingDeps" ] - if [ -n "$rpath" ]; then echo "setting RPATH to: $rpath" >&2 - patchelf --set-rpath "$rpath" "$toPatch" + runPatchelf --set-rpath "$rpath" "$toPatch" fi } @@ -168,10 +183,10 @@ addAutoPatchelfSearchPath() { esac done - cachedDependencies+=( - $(find "$@" "${findOpts[@]}" \! -type d \ - \( -name '*.so' -o -name '*.so.*' \)) - ) + while IFS= read -r -d '' file; do + addToDepCache "$file" + done < <(find "$@" "${findOpts[@]}" \! -type d \ + \( -name '*.so' -o -name '*.so.*' \) -print0) } autoPatchelf() { @@ -197,14 +212,9 @@ autoPatchelf() { echo "automatically fixing dependencies for ELF files" >&2 # Add all shared objects of the current output path to the start of - # cachedDependencies so that it's choosen first in findDependency. + # autoPatchelfCachedDeps so that it's chosen first in findDependency. addAutoPatchelfSearchPath ${norecurse:+--no-recurse} -- "$@" - # Here we actually have a subshell, which also means that - # $cachedDependencies is final at this point, so whenever we want to run - # findDependency outside of this, the dependency cache needs to be rebuilt - # from scratch, so keep this in mind if you want to run findDependency - # outside of this function. while IFS= read -r -d $'\0' file; do isELF "$file" || continue segmentHeaders="$(LANG=C $READELF -l "$file")" @@ -215,8 +225,26 @@ autoPatchelf() { # Skip if the executable is statically linked. [ -n "$(echo "$segmentHeaders" | grep "^ *INTERP\\>")" ] || continue fi + # Jump file if patchelf is unable to parse it + # Some programs contain binary blobs for testing, + # which are identified as ELF but fail to be parsed by patchelf + patchelf "$file" || continue autoPatchelfFile "$file" done < <(find "$@" ${norecurse:+-maxdepth 1} -type f -print0) + + # fail if any dependencies were not found and + # autoPatchelfIgnoreMissingDeps is not set + local depsMissing=0 + for failedDep in "${!autoPatchelfFailedDeps[@]}"; do + echo "autoPatchelfHook could not satisfy dependency $failedDep wanted by ${autoPatchelfFailedDeps[$failedDep]}" + depsMissing=1 + done + # shellcheck disable=SC2154 + # (autoPatchelfIgnoreMissingDeps is referenced but not assigned.) + if [[ $depsMissing == 1 && -z "$autoPatchelfIgnoreMissingDeps" ]]; then + echo "Add the missing dependencies to the build inputs or set autoPatchelfIgnoreMissingDeps=true" + exit 1 + fi } # XXX: This should ultimately use fixupOutputHooks but we currently don't have diff --git a/pkgs/build-support/setup-hooks/compress-man-pages.sh b/pkgs/build-support/setup-hooks/compress-man-pages.sh index 82e48cd8aa7..f5af76e8168 100644 --- a/pkgs/build-support/setup-hooks/compress-man-pages.sh +++ b/pkgs/build-support/setup-hooks/compress-man-pages.sh @@ -21,6 +21,7 @@ compressManPages() { # Point symlinks to compressed manpages. find "$dir"/share/man/ -type l -a '!' -regex '.*\.\(bz2\|gz\)$' -print0 \ + | sort -z \ | while IFS= read -r -d $'\0' f do local target diff --git a/pkgs/build-support/setup-hooks/copy-desktop-items.sh b/pkgs/build-support/setup-hooks/copy-desktop-items.sh new file mode 100644 index 00000000000..f96a10f33d5 --- /dev/null +++ b/pkgs/build-support/setup-hooks/copy-desktop-items.sh @@ -0,0 +1,42 @@ +# shellcheck shell=bash + +# Setup hook that installs specified desktop items. +# +# Example usage in a derivation: +# +# { …, makeDesktopItem, copyDesktopItems, … }: +# +# let desktopItem = makeDesktopItem { … }; in +# stdenv.mkDerivation { +# … +# nativeBuildInputs = [ copyDesktopItems ]; +# +# desktopItems = [ desktopItem ]; +# … +# } +# +# This hook will copy files which are either given by full path +# or all '*.desktop' files placed inside the 'share/applications' +# folder of each `desktopItems` argument. + +postInstallHooks+=(copyDesktopItems) + +copyDesktopItems() { + if [ "${dontCopyDesktopItems-}" = 1 ]; then return; fi + + if [ -z "$desktopItems" ]; then + return + fi + + for desktopItem in $desktopItems; do + if [[ -f "$desktopItem" ]]; then + echo "Copying '$f' into '$out/share/applications'" + install -D -m 444 -t "$out"/share/applications "$f" + else + for f in "$desktopItem"/share/applications/*.desktop; do + echo "Copying '$f' into '$out/share/applications'" + install -D -m 444 -t "$out"/share/applications "$f" + done + fi + done +} diff --git a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh index af2ff0cc966..55e196e654d 100644 --- a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh +++ b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh @@ -23,7 +23,7 @@ fixDarwinDylibNames() { for fn in "$@"; do if [ -L "$fn" ]; then continue; fi echo "$fn: fixing dylib" - int_out=$(install_name_tool -id "$fn" "${flags[@]}" "$fn" 2>&1) + int_out=$(@targetPrefix@install_name_tool -id "$fn" "${flags[@]}" "$fn" 2>&1) result=$? if [ "$result" -ne 0 ] && ! grep "shared library stub file and can't be changed" <<< "$out" diff --git a/pkgs/build-support/setup-hooks/install-shell-files.sh b/pkgs/build-support/setup-hooks/install-shell-files.sh index e0ea1f7f30a..194b408b105 100644 --- a/pkgs/build-support/setup-hooks/install-shell-files.sh +++ b/pkgs/build-support/setup-hooks/install-shell-files.sh @@ -1,4 +1,4 @@ -#!/bin/bash +# shellcheck shell=bash # Setup hook for the `installShellFiles` package. # # Example usage in a derivation: @@ -19,8 +19,8 @@ # installManPage <path> [...<path>] # # Each argument is checked for its man section suffix and installed into the appropriate -# share/man<n>/ directory. The function returns an error if any paths don't have the man section -# suffix (with optional .gz compression). +# share/man/man<n>/ directory. The function returns an error if any paths don't have the man +# section suffix (with optional .gz compression). installManPage() { local path for path in "$@"; do @@ -49,7 +49,7 @@ installManPage() { done } -# installShellCompletion [--bash|--fish|--zsh] ([--name <name>] <path>)... +# installShellCompletion [--cmd <name>] ([--bash|--fish|--zsh] [--name <name>] <path>)... # # Each path is installed into the appropriate directory for shell completions for the given shell. # If one of `--bash`, `--fish`, or `--zsh` is given the path is assumed to belong to that shell. @@ -61,9 +61,20 @@ installManPage() { # If the shell completion needs to be renamed before installing the optional `--name <name>` flag # may be given. Any name provided with this flag only applies to the next path. # +# If all shell completions need to be renamed before installing the optional `--cmd <name>` flag +# may be given. This will synthesize a name for each file, unless overridden with an explicit +# `--name` flag. For example, `--cmd foobar` will synthesize the name `_foobar` for zsh and +# `foobar.bash` for bash. +# # For zsh completions, if the `--name` flag is not given, the path will be automatically renamed # such that `foobar.zsh` becomes `_foobar`. # +# A path may be a named fd, such as produced by the bash construct `<(cmd)`. When using a named fd, +# the shell type flag must be provided, and either the `--name` or `--cmd` flag must be provided. +# This might look something like: +# +# installShellCompletion --zsh --name _foobar <($out/bin/foobar --zsh-completion) +# # This command accepts multiple shell flags in conjunction with multiple paths if you wish to # install them all in one command: # @@ -76,9 +87,16 @@ installManPage() { # installShellCompletion --fish --name foobar.fish share/completions.fish # installShellCompletion --zsh --name _foobar share/completions.zsh # +# Or to use shell newline escaping to split a single invocation across multiple lines: +# +# installShellCompletion --cmd foobar \ +# --bash <($out/bin/foobar --bash-completion) \ +# --fish <($out/bin/foobar --fish-completion) \ +# --zsh <($out/bin/foobar --zsh-completion) +# # If any argument is `--` the remaining arguments will be treated as paths. installShellCompletion() { - local shell='' name='' retval=0 parseArgs=1 arg + local shell='' name='' cmdname='' retval=0 parseArgs=1 arg while { arg=$1; shift; }; do # Parse arguments if (( parseArgs )); then @@ -97,6 +115,17 @@ installShellCompletion() { # treat `--name=foo` the same as `--name foo` name=${arg#--name=} continue;; + --cmd) + cmdname=$1 + shift || { + echo 'installShellCompletion: error: --cmd flag expected an argument' >&2 + return 1 + } + continue;; + --cmd=*) + # treat `--cmd=foo` the same as `--cmd foo` + cmdname=${arg#--cmd=} + continue;; --?*) echo "installShellCompletion: warning: unknown flag ${arg%%=*}" >&2 retval=2 @@ -110,39 +139,67 @@ installShellCompletion() { if (( "${NIX_DEBUG:-0}" >= 1 )); then echo "installShellCompletion: installing $arg${name:+ as $name}" fi - # if we get here, this is a path - # Identify shell - local basename - basename=$(stripHash "$arg") + # if we get here, this is a path or named pipe + # Identify shell and output name local curShell=$shell - if [[ -z "$curShell" ]]; then - # auto-detect the shell - case "$basename" in - ?*.bash) curShell=bash;; - ?*.fish) curShell=fish;; - ?*.zsh) curShell=zsh;; + local outName='' + if [[ -z "$arg" ]]; then + echo "installShellCompletion: error: empty path is not allowed" >&2 + return 1 + elif [[ -p "$arg" ]]; then + # this is a named fd or fifo + if [[ -z "$curShell" ]]; then + echo "installShellCompletion: error: named pipe requires one of --bash, --fish, or --zsh" >&2 + return 1 + elif [[ -z "$name" && -z "$cmdname" ]]; then + echo "installShellCompletion: error: named pipe requires one of --cmd or --name" >&2 + return 1 + fi + else + # this is a path + local argbase + argbase=$(stripHash "$arg") + if [[ -z "$curShell" ]]; then + # auto-detect the shell + case "$argbase" in + ?*.bash) curShell=bash;; + ?*.fish) curShell=fish;; + ?*.zsh) curShell=zsh;; + *) + if [[ "$argbase" = _* && "$argbase" != *.* ]]; then + # probably zsh + echo "installShellCompletion: warning: assuming path \`$arg' is zsh; please specify with --zsh" >&2 + curShell=zsh + else + echo "installShellCompletion: warning: unknown shell for path: $arg" >&2 + retval=2 + continue + fi;; + esac + fi + outName=$argbase + fi + # Identify output path + if [[ -n "$name" ]]; then + outName=$name + elif [[ -n "$cmdname" ]]; then + case "$curShell" in + bash|fish) outName=$cmdname.$curShell;; + zsh) outName=_$cmdname;; *) - if [[ "$basename" = _* && "$basename" != *.* ]]; then - # probably zsh - echo "installShellCompletion: warning: assuming path \`$arg' is zsh; please specify with --zsh" >&2 - curShell=zsh - else - echo "installShellCompletion: warning: unknown shell for path: $arg" >&2 - retval=2 - continue - fi;; + # Our list of shells is out of sync with the flags we accept or extensions we detect. + echo 'installShellCompletion: internal error' >&2 + return 1;; esac fi - # Identify output path - local outName sharePath - outName=${name:-$basename} + local sharePath case "$curShell" in bash) sharePath=bash-completion/completions;; fish) sharePath=fish/vendor_completions.d;; zsh) sharePath=zsh/site-functions # only apply automatic renaming if we didn't have a manual rename - if test -z "$name"; then + if [[ -z "$name" && -z "$cmdname" ]]; then # convert a name like `foo.zsh` into `_foo` outName=${outName%.zsh} outName=_${outName#_} @@ -153,8 +210,16 @@ installShellCompletion() { return 1;; esac # Install file - install -Dm644 -T "$arg" "${!outputBin:?}/share/$sharePath/$outName" || return - # Clear the name, it only applies to one path + local outDir="${!outputBin:?}/share/$sharePath" + local outPath="$outDir/$outName" + if [[ -p "$arg" ]]; then + # install handles named pipes on NixOS but not on macOS + mkdir -p "$outDir" \ + && cat "$arg" > "$outPath" + else + install -Dm644 -T "$arg" "$outPath" + fi || return + # Clear the per-path flags name= done if [[ -n "$name" ]]; then diff --git a/pkgs/build-support/setup-hooks/move-systemd-user-units.sh b/pkgs/build-support/setup-hooks/move-systemd-user-units.sh new file mode 100755 index 00000000000..5963d87c751 --- /dev/null +++ b/pkgs/build-support/setup-hooks/move-systemd-user-units.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# This setup hook, for each output, moves everything in +# $output/lib/systemd/user to $output/share/systemd/user, and replaces +# $output/lib/systemd/user with a symlink to +# $output/share/systemd/user. + +fixupOutputHooks+=(_moveSystemdUserUnits) + +_moveSystemdUserUnits() { + if [ "${dontMoveSystemdUserUnits:-0}" = 1 ]; then return; fi + if [ ! -e "${prefix:?}/lib/systemd/user" ]; then return; fi + local source="$prefix/lib/systemd/user" + local target="$prefix/share/systemd/user" + echo "moving $source/* to $target" + mkdir -p "$target" + ( + shopt -s dotglob + for i in "$source"/*; do + mv "$i" "$target" + done + ) + rmdir "$source" + ln -s "$target" "$source" +} diff --git a/pkgs/build-support/setup-hooks/patch-shebangs.sh b/pkgs/build-support/setup-hooks/patch-shebangs.sh index b48b0c50f57..04ebcd2cc64 100644 --- a/pkgs/build-support/setup-hooks/patch-shebangs.sh +++ b/pkgs/build-support/setup-hooks/patch-shebangs.sh @@ -24,10 +24,10 @@ fixupOutputHooks+=(patchShebangsAuto) patchShebangs() { local pathName - if [ "$1" = "--host" ]; then + if [[ "$1" == "--host" ]]; then pathName=HOST_PATH shift - elif [ "$1" = "--build" ]; then + elif [[ "$1" == "--build" ]]; then pathName=PATH shift fi @@ -41,7 +41,7 @@ patchShebangs() { local oldInterpreterLine local newInterpreterLine - if [ $# -eq 0 ]; then + if [[ $# -eq 0 ]]; then echo "No arguments supplied to patchShebangs" >&2 return 0 fi @@ -50,29 +50,29 @@ patchShebangs() { while IFS= read -r -d $'\0' f; do isScript "$f" || continue - oldInterpreterLine=$(head -1 "$f" | tail -c+3) - read -r oldPath arg0 args <<< "$oldInterpreterLine" + read -r oldInterpreterLine < "$f" + read -r oldPath arg0 args <<< "${oldInterpreterLine:2}" - if [ -z "$pathName" ]; then - if [ -n "$strictDeps" ] && [[ "$f" = "$NIX_STORE"* ]]; then + if [[ -z "$pathName" ]]; then + if [[ -n $strictDeps && $f == "$NIX_STORE"* ]]; then pathName=HOST_PATH else pathName=PATH fi fi - if $(echo "$oldPath" | grep -q "/bin/env$"); then + if [[ "$oldPath" == *"/bin/env" ]]; then # Check for unsupported 'env' functionality: # - options: something starting with a '-' # - environment variables: foo=bar - if $(echo "$arg0" | grep -q -- "^-.*\|.*=.*"); then + if [[ $arg0 == "-"* || $arg0 == *"="* ]]; then echo "$f: unsupported interpreter directive \"$oldInterpreterLine\" (set dontPatchShebangs=1 and handle shebang patching yourself)" >&2 exit 1 fi newPath="$(PATH="${!pathName}" command -v "$arg0" || true)" else - if [ "$oldPath" = "" ]; then + if [[ -z $oldPath ]]; then # If no interpreter is specified linux will use /bin/sh. Set # oldpath="/bin/sh" so that we get /nix/store/.../sh. oldPath="/bin/sh" @@ -84,19 +84,19 @@ patchShebangs() { fi # Strip trailing whitespace introduced when no arguments are present - newInterpreterLine="$(echo "$newPath $args" | sed 's/[[:space:]]*$//')" + newInterpreterLine="$newPath $args" + newInterpreterLine=${newInterpreterLine%${newInterpreterLine##*[![:space:]]}} - if [ -n "$oldPath" -a "${oldPath:0:${#NIX_STORE}}" != "$NIX_STORE" ]; then - if [ -n "$newPath" -a "$newPath" != "$oldPath" ]; then + if [[ -n "$oldPath" && "${oldPath:0:${#NIX_STORE}}" != "$NIX_STORE" ]]; then + if [[ -n "$newPath" && "$newPath" != "$oldPath" ]]; then echo "$f: interpreter directive changed from \"$oldInterpreterLine\" to \"$newInterpreterLine\"" # escape the escape chars so that sed doesn't interpret them - escapedInterpreterLine=$(echo "$newInterpreterLine" | sed 's|\\|\\\\|g') + escapedInterpreterLine=${newInterpreterLine//\\/\\\\} + # Preserve times, see: https://github.com/NixOS/nixpkgs/pull/33281 - timestamp=$(mktemp) - touch -r "$f" "$timestamp" + timestamp=$(stat --printf "%y" "$f") sed -i -e "1 s|.*|#\!$escapedInterpreterLine|" "$f" - touch -r "$timestamp" "$f" - rm "$timestamp" + touch --date "$timestamp" "$f" fi fi done < <(find "$@" -type f -perm -0100 -print0) @@ -105,12 +105,12 @@ patchShebangs() { } patchShebangsAuto () { - if [ -z "${dontPatchShebangs-}" -a -e "$prefix" ]; then + if [[ -z "${dontPatchShebangs-}" && -e "$prefix" ]]; then # Dev output will end up being run on the build platform. An # example case of this is sdl2-config. Otherwise, we can just # use the runtime path (--host). - if [ "$output" != out ] && [ "$output" = "$outputDev" ]; then + if [[ "$output" != out && "$output" = "$outputDev" ]]; then patchShebangs --build "$prefix" else patchShebangs --host "$prefix" diff --git a/pkgs/build-support/setup-hooks/reproducible-builds.sh b/pkgs/build-support/setup-hooks/reproducible-builds.sh new file mode 100644 index 00000000000..5b01c213fe4 --- /dev/null +++ b/pkgs/build-support/setup-hooks/reproducible-builds.sh @@ -0,0 +1,9 @@ +# Use the last part of the out path as hash input for the build. +# This should ensure that it is deterministic across rebuilds of the same +# derivation and not easily collide with other builds. +# We also truncate the hash so that it cannot cause reference cycles. +export NIX_CFLAGS_COMPILE+=" -frandom-seed=$( + outbase="${out##*/}" + randomseed="${outbase:0:10}" + echo $randomseed +)" diff --git a/pkgs/build-support/setup-hooks/strip.sh b/pkgs/build-support/setup-hooks/strip.sh index f5fa9378fd7..c31a50eba57 100644 --- a/pkgs/build-support/setup-hooks/strip.sh +++ b/pkgs/build-support/setup-hooks/strip.sh @@ -51,7 +51,7 @@ stripDirs() { if [ -n "${dirs}" ]; then header "stripping (with command $cmd and flags $stripFlags) in$dirs" - find $dirs -type f -print0 | xargs -0 ${xargsFlags:--r} $cmd $commonStripFlags $stripFlags 2>/dev/null || true + find $dirs -type f -exec $cmd $commonStripFlags $stripFlags '{}' \; 2>/dev/null stopNest fi } diff --git a/pkgs/build-support/setup-hooks/validate-pkg-config.sh b/pkgs/build-support/setup-hooks/validate-pkg-config.sh index 54fc9cc122c..ada1b56760d 100644 --- a/pkgs/build-support/setup-hooks/validate-pkg-config.sh +++ b/pkgs/build-support/setup-hooks/validate-pkg-config.sh @@ -3,9 +3,8 @@ fixupOutputHooks+=(_validatePkgConfig) _validatePkgConfig() { + local bail=0 for pc in $(find "$prefix" -name '*.pc'); do - local bail=0 - # Do not fail immediately. It's nice to see all errors when # there are multiple pkgconfig files. if ! pkg-config --validate "$pc"; then diff --git a/pkgs/build-support/setup-hooks/wrap-gapps-hook/default.nix b/pkgs/build-support/setup-hooks/wrap-gapps-hook/default.nix index 5a87893d972..d0ea088bf71 100644 --- a/pkgs/build-support/setup-hooks/wrap-gapps-hook/default.nix +++ b/pkgs/build-support/setup-hooks/wrap-gapps-hook/default.nix @@ -3,6 +3,7 @@ , makeSetupHook , makeWrapper , gobject-introspection +, isGraphical ? true , gtk3 , librsvg , dconf @@ -21,7 +22,7 @@ makeSetupHook { # Unfortunately, it also requires the user to have dconf # D-Bus service enabled globally (e.g. through a NixOS module). dconf.lib - ] ++ [ + ] ++ lib.optionals isGraphical [ # TODO: remove this, packages should depend on GTK explicitly. gtk3 @@ -30,6 +31,7 @@ makeSetupHook { # graphics in GTK (e.g. cross for closing window in window title bar) # so it is pretty much required for applications using GTK. librsvg + ] ++ [ # We use the wrapProgram function. makeWrapper diff --git a/pkgs/build-support/setup-systemd-units.nix b/pkgs/build-support/setup-systemd-units.nix index 4fa2f42c39d..4c7ee86669f 100644 --- a/pkgs/build-support/setup-systemd-units.nix +++ b/pkgs/build-support/setup-systemd-units.nix @@ -58,7 +58,7 @@ unitDir=/etc/systemd/system if [ ! -w "$unitDir" ]; then - unitDir=/etc/systemd-mutable/system + unitDir=/nix/var/nix/profiles/default/lib/systemd/system mkdir -p "$unitDir" fi declare -a unitsToStop unitsToStart diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix index f1d4808e3cc..318f5b430fe 100644 --- a/pkgs/build-support/singularity-tools/default.nix +++ b/pkgs/build-support/singularity-tools/default.nix @@ -1,4 +1,5 @@ { runCommand +, lib , stdenv , storeDir ? builtins.storeDir , writeScript @@ -7,7 +8,7 @@ , bash , vmTools , gawk -, utillinux +, util-linux , runtimeShell , e2fsprogs }: @@ -47,7 +48,7 @@ rec { runScriptFile = shellScript "run-script.sh" runScript; result = vmTools.runInLinuxVM ( runCommand "singularity-image-${name}.img" { - buildInputs = [ singularity e2fsprogs utillinux gawk ]; + buildInputs = [ singularity e2fsprogs util-linux gawk ]; layerClosure = writeReferencesToFile layer; preVM = vmTools.createEmptyImage { size = diskSize; @@ -64,7 +65,7 @@ rec { mkdir proc sys dev # Run root script - ${stdenv.lib.optionalString (runAsRoot != null) '' + ${lib.optionalString (runAsRoot != null) '' mkdir -p ./${storeDir} mount --rbind ${storeDir} ./${storeDir} unshare -imnpuf --mount-proc chroot ./ ${runAsRootFile} @@ -86,7 +87,9 @@ rec { done # Create runScript and link shell - ln -s ${runtimeShell} bin/sh + if [ ! -e bin/sh ]; then + ln -s ${runtimeShell} bin/sh + fi mkdir -p .singularity.d ln -s ${runScriptFile} .singularity.d/runscript diff --git a/pkgs/build-support/skaware/build-skaware-package.nix b/pkgs/build-support/skaware/build-skaware-package.nix index e6e2e35789b..b27b65f48a5 100644 --- a/pkgs/build-support/skaware/build-skaware-package.nix +++ b/pkgs/build-support/skaware/build-skaware-package.nix @@ -1,6 +1,5 @@ -{ stdenv, cleanPackaging, fetchurl }: -let lib = stdenv.lib; -in { +{ lib, stdenv, cleanPackaging, fetchurl }: +{ # : string pname # : string @@ -98,9 +97,9 @@ in stdenv.mkDerivation { meta = { homepage = "https://skarnet.org/software/${pname}/"; inherit description platforms; - license = stdenv.lib.licenses.isc; + license = lib.licenses.isc; maintainers = with lib.maintainers; - [ pmahoney Profpatsch ] ++ maintainers; + [ pmahoney Profpatsch qyliss ] ++ maintainers; }; } diff --git a/pkgs/build-support/skaware/clean-packaging.nix b/pkgs/build-support/skaware/clean-packaging.nix index 762fe25c0ac..d51cbec8aeb 100644 --- a/pkgs/build-support/skaware/clean-packaging.nix +++ b/pkgs/build-support/skaware/clean-packaging.nix @@ -3,12 +3,12 @@ # files were either discarded or moved to outputs. # This ensures nothing is forgotten and new files # are correctly handled on update. -{ stdenv, file, writeScript }: +{ lib, stdenv, file, writeScript }: let - globWith = stdenv.lib.concatMapStringsSep "\n"; + globWith = lib.concatMapStringsSep "\n"; rmNoise = noiseGlobs: globWith (f: - ''rm -rf ${f}'') noiseGlobs; + "rm -rf ${f}") noiseGlobs; mvDoc = docGlobs: globWith (f: ''mv ${f} "$DOCDIR" 2>/dev/null || true'') docGlobs; diff --git a/pkgs/build-support/substitute-files/substitute-all-files.nix b/pkgs/build-support/substitute-files/substitute-all-files.nix index 66feb695c41..682e976dcfe 100644 --- a/pkgs/build-support/substitute-files/substitute-all-files.nix +++ b/pkgs/build-support/substitute-files/substitute-all-files.nix @@ -1,10 +1,10 @@ -{ stdenv }: +{ lib, stdenv }: args: stdenv.mkDerivation ({ name = if args ? name then args.name else baseNameOf (toString args.src); - builder = with stdenv.lib; builtins.toFile "builder.sh" '' + builder = builtins.toFile "builder.sh" '' source $stdenv/setup set -o pipefail @@ -13,7 +13,7 @@ stdenv.mkDerivation ({ args= pushd "$src" - echo -ne "${concatStringsSep "\\0" args.files}" | xargs -0 -n1 -I {} -- find {} -type f -print0 | while read -d "" line; do + echo -ne "${lib.concatStringsSep "\\0" args.files}" | xargs -0 -n1 -I {} -- find {} -type f -print0 | while read -d "" line; do mkdir -p "$out/$(dirname "$line")" substituteAll "$line" "$out/$line" done diff --git a/pkgs/build-support/templaterpm/default.nix b/pkgs/build-support/templaterpm/default.nix index 30465c740e5..efe70efe6c4 100644 --- a/pkgs/build-support/templaterpm/default.nix +++ b/pkgs/build-support/templaterpm/default.nix @@ -1,10 +1,11 @@ -{stdenv, makeWrapper, python, toposort, rpm}: +{lib, stdenv, makeWrapper, python, toposort, rpm}: stdenv.mkDerivation { pname = "nix-template-rpm"; version = "0.1"; - buildInputs = [ makeWrapper python toposort rpm ]; + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ python toposort rpm ]; phases = [ "installPhase" "fixupPhase" ]; @@ -15,10 +16,10 @@ stdenv.mkDerivation { --set PYTHONPATH "${rpm}/lib/${python.libPrefix}/site-packages":"${toposort}/lib/${python.libPrefix}/site-packages" ''; - meta = with stdenv.lib; { + meta = with lib; { description = "Create templates of nix expressions from RPM .spec files"; maintainers = with maintainers; [ tstrobel ]; - platforms = with stdenv.lib.platforms; unix; + platforms = platforms.unix; hydraPlatforms = []; }; } diff --git a/pkgs/build-support/templaterpm/nix-template-rpm.py b/pkgs/build-support/templaterpm/nix-template-rpm.py index f39595f8977..db8c0f2064c 100755 --- a/pkgs/build-support/templaterpm/nix-template-rpm.py +++ b/pkgs/build-support/templaterpm/nix-template-rpm.py @@ -229,26 +229,26 @@ class SPECTemplate(object): @property def meta(self): - out = ' meta = {\n' + out = ' meta = with lib; {\n' out += ' homepage = ' + self.spec.sourceHeader['url'] + ';\n' out += ' description = "' + self.spec.sourceHeader['summary'] + '";\n' - out += ' license = stdenv.lib.licenses.' + self.spec.sourceHeader['license'] + ';\n' + out += ' license = lib.licenses.' + self.spec.sourceHeader['license'] + ';\n' out += ' platforms = [ "i686-linux" "x86_64-linux" ];\n' - out += ' maintainers = with stdenv.lib.maintainers; [ ' + self.maintainer + ' ];\n' + out += ' maintainers = with lib.maintainers; [ ' + self.maintainer + ' ];\n' out += ' };\n' out += '}\n' return out def __str__(self): - head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' + head = '{lib, stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' head += 'stdenv.mkDerivation {\n' body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ] return head + '\n'.join(body) def getTemplate(self): - head = '{stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' + head = '{lib, stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n' head += 'let\n' head += ' buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n' head += 'in\n\n' diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index eab5366e183..6f51ba512c1 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -1,27 +1,12 @@ { lib, stdenv, stdenvNoCC, lndir, runtimeShell }: -let - - runCommand' = runLocal: stdenv: name: env: buildCommand: - stdenv.mkDerivation ({ - name = lib.strings.sanitizeDerivationName name; - inherit buildCommand; - passAsFile = [ "buildCommand" ]; - } - // (lib.optionalAttrs runLocal { - preferLocalBuild = true; - allowSubstitutes = false; - }) - // env); - -in - rec { /* Run the shell command `buildCommand' to produce a store path named * `name'. The attributes in `env' are added to the environment - * prior to running the command. By default `runCommand' runs using - * stdenv with no compiler environment. `runCommandCC` + * prior to running the command. By default `runCommand` runs in a + * stdenv with no compiler environment. `runCommandCC` uses the default + * stdenv, `pkgs.stdenv`. * * Examples: * runCommand "name" {envVariable = true;} ''echo hello > $out'' @@ -42,13 +27,64 @@ rec { runCommand = runCommandNoCC; runCommandLocal = runCommandNoCCLocal; - runCommandNoCC = runCommand' false stdenvNoCC; - runCommandNoCCLocal = runCommand' true stdenvNoCC; + runCommandNoCC = name: env: runCommandWith { + stdenv = stdenvNoCC; + runLocal = false; + inherit name; + derivationArgs = env; + }; + runCommandNoCCLocal = name: env: runCommandWith { + stdenv = stdenvNoCC; + runLocal = true; + inherit name; + derivationArgs = env; + }; - runCommandCC = runCommand' false stdenv; + runCommandCC = name: env: runCommandWith { + stdenv = stdenv; + runLocal = false; + inherit name; + derivationArgs = env; + }; # `runCommandCCLocal` left out on purpose. # We shouldn’t force the user to have a cc in scope. + /* Generalized version of the `runCommand`-variants + * which does customized behavior via a single + * attribute set passed as the first argument + * instead of having a lot of variants like + * `runCommand*`. Additionally it allows changing + * the used `stdenv` freely and has a more explicit + * approach to changing the arguments passed to + * `stdenv.mkDerivation`. + */ + runCommandWith = + let + # prevent infinite recursion for the default stdenv value + defaultStdenv = stdenv; + in + { stdenv ? defaultStdenv + # which stdenv to use, defaults to a stdenv with a C compiler, pkgs.stdenv + , runLocal ? false + # whether to build this derivation locally instead of substituting + , derivationArgs ? {} + # extra arguments to pass to stdenv.mkDerivation + , name + # name of the resulting derivation + }: buildCommand: + stdenv.mkDerivation ({ + name = lib.strings.sanitizeDerivationName name; + inherit buildCommand; + passAsFile = [ "buildCommand" ] + ++ (derivationArgs.passAsFile or []); + } + // (lib.optionalAttrs runLocal { + preferLocalBuild = true; + allowSubstitutes = false; + }) + // builtins.removeAttrs derivationArgs [ "passAsFile" ]); + + /* Writes a text file to the nix store. * The contents of text is added to the file in the store. * @@ -80,7 +116,7 @@ rec { , checkPhase ? "" # syntax checks, e.g. for scripts }: runCommand name - { inherit text executable; + { inherit text executable checkPhase; passAsFile = [ "text" ]; # Pointless to do this on a remote machine. preferLocalBuild = true; @@ -96,7 +132,7 @@ rec { echo -n "$text" > "$n" fi - ${checkPhase} + eval "$checkPhase" (test -n "$executable" && chmod +x "$n") || true ''; @@ -402,6 +438,35 @@ rec { done < graph ''; + /* + Write the set of references to a file, that is, their immediate dependencies. + + This produces the equivalent of `nix-store -q --references`. + */ + writeDirectReferencesToFile = path: runCommand "runtime-references" + { + exportReferencesGraph = ["graph" path]; + inherit path; + } + '' + touch ./references + while read p; do + read dummy + read nrRefs + if [[ $p == $path ]]; then + for ((i = 0; i < nrRefs; i++)); do + read ref; + echo $ref >>./references + done + else + for ((i = 0; i < nrRefs; i++)); do + read ref; + done + fi + done < graph + sort ./references >$out + ''; + /* Print an error message if the file with the specified name and * hash doesn't exist in the Nix store. This function should only @@ -505,4 +570,53 @@ rec { phases = "unpackPhase patchPhase installPhase"; installPhase = "cp -R ./ $out"; }; + + /* An immutable file in the store with a length of 0 bytes. */ + emptyFile = runCommand "empty-file" { + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "0ip26j2h11n1kgkz36rl4akv694yz65hr72q4kv4b3lxcbi65b3p"; + preferLocalBuild = true; + } "touch $out"; + + /* An immutable empty directory in the store. */ + emptyDirectory = runCommand "empty-directory" { + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; + preferLocalBuild = true; + } "mkdir $out"; + + /* Checks the command output contains the specified version + * + * Although simplistic, this test assures that the main program + * can run. While there's no substitute for a real test case, + * it does catch dynamic linking errors and such. It also provides + * some protection against accidentally building the wrong version, + * for example when using an 'old' hash in a fixed-output derivation. + * + * Examples: + * + * passthru.tests.version = testVersion { package = hello; }; + * + * passthru.tests.version = testVersion { + * package = seaweedfs; + * command = "weed version"; + * }; + * + * passthru.tests.version = testVersion { + * package = key; + * command = "KeY --help"; + * # Wrong '2.5' version in the code. Drop on next version. + * version = "2.5"; + * }; + */ + testVersion = + { package, + command ? "${package.meta.mainProgram or package.pname or package.name} --version", + version ? package.version, + }: runCommand "test-version" { nativeBuildInputs = [ package ]; meta.timeout = 60; } '' + ${command} | grep -Fw ${version} + touch $out + ''; } diff --git a/pkgs/build-support/trivial-builders/test-overriding.nix b/pkgs/build-support/trivial-builders/test-overriding.nix new file mode 100644 index 00000000000..ddd5dc05075 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test-overriding.nix @@ -0,0 +1,119 @@ +# Check that overriding works for trivial-builders like +# `writeShellScript` via `overrideAttrs`. This is useful +# to override the `checkPhase`, e. g. when you want +# to enable extglob in `writeShellScript`. +# +# Run using `nix-build -A tests.trivial-overriding`. +{ lib +, runtimeShell +, runCommand +, callPackage +, writeShellScript +, writeTextFile +, writeShellScriptBin +}: + +let + extglobScript = '' + shopt -s extglob + touch success + echo @(success|failure) + rm success + ''; + + # Reuse the old `checkPhase` of `writeShellScript`, but enable extglob. + allowExtglob = old: { + checkPhase = '' + # make sure we don't change the settings for + # the rest of the derivation's build + ( + export BASHOPTS + shopt -s extglob + ${old.checkPhase} + ) + ''; + }; + + # Run old checkPhase, but only succeed if it fails. + # This HACK is required because we can't introspect build failures + # in nix: With `assertFail` we want to make sure that the default + # `checkPhase` would fail if extglob was used in the script. + assertFail = old: { + # write old checkPhase into a shell script, so we can check for + # the phase to fail even though we have `set -e`. + checkPhase = '' + if source ${writeShellScript "old-check-phase" old.checkPhase} 2>/dev/null; then + exit 1 + fi + ''; + }; + + simpleCase = case: + writeShellScript "test-trivial-overriding-${case}" extglobScript; + + callPackageCase = case: callPackage ( + { writeShellScript }: + writeShellScript "test-trivial-callpackage-overriding-${case}" extglobScript + ) { }; + + binCase = case: + writeShellScriptBin "test-trivial-overriding-bin-${case}" extglobScript; + + # building this derivation would fail without overriding + textFileCase = writeTextFile { + name = "test-trivial-overriding-text-file"; + checkPhase = "false"; + text = '' + #!${runtimeShell} + echo success + ''; + executable = true; + }; + + mkCase = f: type: isBin: + let + drv = (f type).overrideAttrs + (if type == "succ" then allowExtglob else assertFail); + in if isBin then "${drv}/bin/${drv.name}" else drv; + + writeTextOverrides = { + # Enabling globbing in checkPhase + simpleSucc = mkCase simpleCase "succ" false; + # Ensure it's possible to fail; in this case globbing is not enabled. + simpleFail = mkCase simpleCase "fail" false; + # Do the same checks after wrapping with callPackage + # to make sure callPackage doesn't mess with the override + callpSucc = mkCase callPackageCase "succ" false; + callpFail = mkCase callPackageCase "fail" false; + # Do the same check using `writeShellScriptBin` + binSucc = mkCase binCase "succ" true; + binFail = mkCase binCase "fail" true; + # Check that we can also override plain writeTextFile + textFileSuccess = textFileCase.overrideAttrs (_: { + checkPhase = "true"; + }); + }; + + # `runTest` forces nix to build the script of our test case and + # run its `checkPhase` which is our main interest. Additionally + # it executes the script and thus makes sure that extglob also + # works at run time. + runTest = script: + let + name = script.name or (builtins.baseNameOf script); + in writeShellScript "run-${name}" '' + if [ "$(${script})" != "success" ]; then + echo "Failed in ${script}" + exit 1 + fi + ''; +in + +runCommand "test-writeShellScript-overriding" { + passthru = { inherit writeTextOverrides; }; +} '' + ${lib.concatMapStrings (test: '' + ${runTest test} + '') (lib.attrValues writeTextOverrides)} + touch "$out" +'' diff --git a/pkgs/build-support/trivial-builders/test.nix b/pkgs/build-support/trivial-builders/test.nix new file mode 100644 index 00000000000..204fb54fca3 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test.nix @@ -0,0 +1,53 @@ +{ lib, nixosTest, pkgs, writeText, hello, figlet, stdenvNoCC }: + +# -------------------------------------------------------------------------- # +# +# trivial-builders test +# +# -------------------------------------------------------------------------- # +# +# This file can be run independently (quick): +# +# $ pkgs/build-support/trivial-builders/test.sh +# +# or in the build sandbox with a ~20s VM overhead +# +# $ nix-build -A tests.trivial-builders +# +# -------------------------------------------------------------------------- # + +let + invokeSamples = file: + lib.concatStringsSep " " ( + lib.attrValues (import file { inherit pkgs; }) + ); +in +nixosTest { + name = "nixpkgs-trivial-builders"; + nodes.machine = { ... }: { + virtualisation.writableStore = true; + + # Test runs without network, so we don't substitute and prepare our deps + nix.binaryCaches = lib.mkForce []; + environment.etc."pre-built-paths".source = writeText "pre-built-paths" ( + builtins.toJSON [hello figlet stdenvNoCC] + ); + environment.variables = { + SAMPLE = invokeSamples ./test/sample.nix; + REFERENCES = invokeSamples ./test/invoke-writeReferencesToFile.nix; + DIRECT_REFS = invokeSamples ./test/invoke-writeDirectReferencesToFile.nix; + }; + }; + testScript = '' + machine.succeed(""" + ${./test.sh} 2>/dev/console + """) + ''; + meta = { + license = lib.licenses.mit; # nixpkgs license + maintainers = with lib.maintainers; [ + roberth + ]; + description = "Run the Nixpkgs trivial builders tests"; + }; +} diff --git a/pkgs/build-support/trivial-builders/test.sh b/pkgs/build-support/trivial-builders/test.sh new file mode 100755 index 00000000000..b7c4726a9be --- /dev/null +++ b/pkgs/build-support/trivial-builders/test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# -------------------------------------------------------------------------- # +# +# trivial-builders test +# +# -------------------------------------------------------------------------- # +# +# This file can be run independently (quick): +# +# $ pkgs/build-support/trivial-builders/test.sh +# +# or in the build sandbox with a ~20s VM overhead +# +# $ nix-build -A tests.trivial-builders +# +# -------------------------------------------------------------------------- # + +# strict bash +set -euo pipefail + +# debug +# set -x +# PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +cd "$(dirname ${BASH_SOURCE[0]})" # nixpkgs root + +if [[ -z ${SAMPLE:-} ]]; then + sample=( `nix-build test/sample.nix` ) + directRefs=( `nix-build test/invoke-writeDirectReferencesToFile.nix` ) + references=( `nix-build test/invoke-writeReferencesToFile.nix` ) +else + # Injected by Nix (to avoid evaluating in a derivation) + # turn them into arrays + sample=($SAMPLE) + directRefs=($DIRECT_REFS) + references=($REFERENCES) +fi + +echo >&2 Testing direct references... +for i in "${!sample[@]}"; do + echo >&2 Checking '#'$i ${sample[$i]} ${directRefs[$i]} + diff -U3 \ + <(sort <${directRefs[$i]}) \ + <(nix-store -q --references ${sample[$i]} | sort) +done + +echo >&2 Testing closure... +for i in "${!sample[@]}"; do + echo >&2 Checking '#'$i ${sample[$i]} ${references[$i]} + diff -U3 \ + <(sort <${references[$i]}) \ + <(nix-store -q --requisites ${sample[$i]} | sort) +done + +echo 'OK!' diff --git a/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix b/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix new file mode 100644 index 00000000000..ead3f7a2f57 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/invoke-writeDirectReferencesToFile.nix @@ -0,0 +1,4 @@ +{ pkgs ? import ../../../.. { config = {}; overlays = []; } }: +pkgs.lib.mapAttrs + (k: v: pkgs.writeDirectReferencesToFile v) + (import ./sample.nix { inherit pkgs; }) diff --git a/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix b/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix new file mode 100644 index 00000000000..99c6c2f7dcc --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/invoke-writeReferencesToFile.nix @@ -0,0 +1,4 @@ +{ pkgs ? import ../../../.. { config = {}; overlays = []; } }: +pkgs.lib.mapAttrs + (k: v: pkgs.writeReferencesToFile v) + (import ./sample.nix { inherit pkgs; }) diff --git a/pkgs/build-support/trivial-builders/test/sample.nix b/pkgs/build-support/trivial-builders/test/sample.nix new file mode 100644 index 00000000000..27aee6b73db --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/sample.nix @@ -0,0 +1,19 @@ +{ pkgs ? import ../../../.. { config = {}; overlays = []; } }: +let + inherit (pkgs) + figlet + hello + writeText + ; +in +{ + hello = hello; + figlet = figlet; + norefs = writeText "hi" "hello"; + helloRef = writeText "hi" "hello ${hello}"; + helloFigletRef = writeText "hi" "hello ${hello} ${figlet}"; + inherit (pkgs) + emptyFile + emptyDirectory + ; +} diff --git a/pkgs/build-support/vm/deb/deb-closure.pl b/pkgs/build-support/vm/deb/deb-closure.pl index bed397d6f07..fe23025df1d 100644 --- a/pkgs/build-support/vm/deb/deb-closure.pl +++ b/pkgs/build-support/vm/deb/deb-closure.pl @@ -50,7 +50,7 @@ sub getDeps { # virtual dependencies. my %provides; -foreach my $cdata (values %packages) { +foreach my $cdata (sort {$a->{Package} cmp $b->{Package}} (values %packages)) { if (defined $cdata->{Provides}) { my @provides = getDeps(Dpkg::Deps::deps_parse($cdata->{Provides})); foreach my $name (@provides) { diff --git a/pkgs/build-support/vm/default.nix b/pkgs/build-support/vm/default.nix index 909cdc6da04..cfc19c03cfd 100644 --- a/pkgs/build-support/vm/default.nix +++ b/pkgs/build-support/vm/default.nix @@ -1,6 +1,7 @@ -{ pkgs +{ lib +, pkgs , kernel ? pkgs.linux -, img ? pkgs.stdenv.hostPlatform.platform.kernelTarget +, img ? pkgs.stdenv.hostPlatform.linux-kernel.target , storeDir ? builtins.storeDir , rootModules ? [ "virtio_pci" "virtio_mmio" "virtio_blk" "virtio_balloon" "virtio_rng" "ext4" "unix" "9p" "9pnet_virtio" "crc32c_generic" ] @@ -151,7 +152,7 @@ rec { # Set the system time from the hardware clock. Works around an # apparent KVM > 1.5.2 bug. - ${pkgs.utillinux}/bin/hwclock -s + ${pkgs.util-linux}/bin/hwclock -s export NIX_STORE=${storeDir} export NIX_BUILD_TOP=/tmp @@ -270,7 +271,7 @@ rec { defaultCreateRootFS = '' mkdir /mnt ${e2fsprogs}/bin/mkfs.ext4 /dev/${hd} - ${utillinux}/bin/mount -t ext4 /dev/${hd} /mnt + ${util-linux}/bin/mount -t ext4 /dev/${hd} /mnt if test -e /mnt/.debug; then exec ${bash}/bin/sh @@ -317,7 +318,7 @@ rec { with pkgs; runInLinuxVM ( stdenv.mkDerivation { name = "extract-file"; - buildInputs = [ utillinux ]; + buildInputs = [ util-linux ]; buildCommand = '' ln -s ${kernel}/lib /lib ${kmod}/bin/modprobe loop @@ -342,7 +343,7 @@ rec { with pkgs; runInLinuxVM ( stdenv.mkDerivation { name = "extract-file-mtd"; - buildInputs = [ utillinux mtdutils ]; + buildInputs = [ util-linux mtdutils ]; buildCommand = '' ln -s ${kernel}/lib /lib ${kmod}/bin/modprobe mtd @@ -417,7 +418,7 @@ rec { # Make the Nix store available in /mnt, because that's where the RPMs live. mkdir -p /mnt${storeDir} - ${utillinux}/bin/mount -o bind ${storeDir} /mnt${storeDir} + ${util-linux}/bin/mount -o bind ${storeDir} /mnt${storeDir} # Newer distributions like Fedora 18 require /lib etc. to be # symlinked to /usr. @@ -427,7 +428,7 @@ rec { ln -s /usr/sbin /mnt/sbin ln -s /usr/lib /mnt/lib ln -s /usr/lib64 /mnt/lib64 - ${utillinux}/bin/mount -t proc none /mnt/proc + ${util-linux}/bin/mount -t proc none /mnt/proc ''} echo "unpacking RPMs..." @@ -445,7 +446,7 @@ rec { PATH=/usr/bin:/bin:/usr/sbin:/sbin $chroot /mnt \ rpm --initdb - ${utillinux}/bin/mount -o bind /tmp /mnt/tmp + ${util-linux}/bin/mount -o bind /tmp /mnt/tmp echo "installing RPMs..." PATH=/usr/bin:/bin:/usr/sbin:/sbin $chroot /mnt \ @@ -456,8 +457,8 @@ rec { rm /mnt/.debug - ${utillinux}/bin/umount /mnt${storeDir} /mnt/tmp ${lib.optionalString unifiedSystemDir "/mnt/proc"} - ${utillinux}/bin/umount /mnt + ${util-linux}/bin/umount /mnt${storeDir} /mnt/tmp ${lib.optionalString unifiedSystemDir "/mnt/proc"} + ${util-linux}/bin/umount /mnt ''; passthru = { inherit fullName; }; @@ -572,7 +573,7 @@ rec { buildCommand = '' ${createRootFS} - PATH=$PATH:${stdenv.lib.makeBinPath [ dpkg dpkg glibc lzma ]} + PATH=$PATH:${lib.makeBinPath [ dpkg dpkg glibc xz ]} # Unpack the .debs. We do this to prevent pre-install scripts # (which have lots of circular dependencies) from barfing. @@ -587,9 +588,9 @@ rec { # Make the Nix store available in /mnt, because that's where the .debs live. mkdir -p /mnt/inst${storeDir} - ${utillinux}/bin/mount -o bind ${storeDir} /mnt/inst${storeDir} - ${utillinux}/bin/mount -o bind /proc /mnt/proc - ${utillinux}/bin/mount -o bind /dev /mnt/dev + ${util-linux}/bin/mount -o bind ${storeDir} /mnt/inst${storeDir} + ${util-linux}/bin/mount -o bind /proc /mnt/proc + ${util-linux}/bin/mount -o bind /dev /mnt/dev # Misc. files/directories assumed by various packages. echo "initialising Dpkg DB..." @@ -635,10 +636,10 @@ rec { rm /mnt/.debug - ${utillinux}/bin/umount /mnt/inst${storeDir} - ${utillinux}/bin/umount /mnt/proc - ${utillinux}/bin/umount /mnt/dev - ${utillinux}/bin/umount /mnt + ${util-linux}/bin/umount /mnt/inst${storeDir} + ${util-linux}/bin/umount /mnt/proc + ${util-linux}/bin/umount /mnt/dev + ${util-linux}/bin/umount /mnt ''; passthru = { inherit fullName; }; @@ -821,45 +822,6 @@ rec { /* The set of supported Dpkg-based distributions. */ debDistros = { - - # Interestingly, the SHA-256 hashes provided by Ubuntu in - # http://nl.archive.ubuntu.com/ubuntu/dists/{gutsy,hardy}/Release are - # wrong, but the SHA-1 and MD5 hashes are correct. Intrepid is fine. - - ubuntu1204i386 = { - name = "ubuntu-12.04-precise-i386"; - fullName = "Ubuntu 12.04 Precise (i386)"; - packagesLists = - [ (fetchurl { - url = "mirror://ubuntu/dists/precise/main/binary-i386/Packages.bz2"; - sha256 = "18ns9h4qhvjfcip9z55grzi371racxavgqkp6b5kfkdq2wwwax2d"; - }) - (fetchurl { - url = "mirror://ubuntu/dists/precise/universe/binary-i386/Packages.bz2"; - sha256 = "085lkzbnzkc74kfdmwdc32sfqyfz8dr0rbiifk8kx9jih3xjw2jk"; - }) - ]; - urlPrefix = "mirror://ubuntu"; - packages = commonDebPackages ++ [ "diffutils" ]; - }; - - ubuntu1204x86_64 = { - name = "ubuntu-12.04-precise-amd64"; - fullName = "Ubuntu 12.04 Precise (amd64)"; - packagesLists = - [ (fetchurl { - url = "mirror://ubuntu/dists/precise/main/binary-amd64/Packages.bz2"; - sha256 = "1aabpn0hdih6cbabyn87yvhccqj44q9k03mqmjsb920iqlckl3fc"; - }) - (fetchurl { - url = "mirror://ubuntu/dists/precise/universe/binary-amd64/Packages.bz2"; - sha256 = "0x4hz5aplximgb7gnpvrhkw8m7a40s80rkm5b8hil0afblwlg4vr"; - }) - ]; - urlPrefix = "mirror://ubuntu"; - packages = commonDebPackages ++ [ "diffutils" ]; - }; - ubuntu1404i386 = { name = "ubuntu-14.04-trusty-i386"; fullName = "Ubuntu 14.04 Trusty (i386)"; @@ -928,119 +890,117 @@ rec { packages = commonDebPackages ++ [ "diffutils" "libc-bin" ]; }; - ubuntu1710i386 = { - name = "ubuntu-17.10-artful-i386"; - fullName = "Ubuntu 17.10 Artful (i386)"; + ubuntu1804i386 = { + name = "ubuntu-18.04-bionic-i386"; + fullName = "Ubuntu 18.04 Bionic (i386)"; packagesLists = [ (fetchurl { - url = "mirror://ubuntu/dists/artful/main/binary-i386/Packages.xz"; - sha256 = "18yrj4kqdzm39q0527m97h5ing58hkm9yq9iyj636zh2rclym3c8"; + url = "mirror://ubuntu/dists/bionic/main/binary-i386/Packages.xz"; + sha256 = "0f0v4131kwf7m7f8j3288rlqdxk1k3vqy74b7fcfd6jz9j8d840i"; }) (fetchurl { - url = "mirror://ubuntu/dists/artful/universe/binary-i386/Packages.xz"; - sha256 = "1v0njw2w80xfmxi7by76cs8hyxlla5h3gqajlpdw5srjgx2qrm2g"; + url = "mirror://ubuntu/dists/bionic/universe/binary-i386/Packages.xz"; + sha256 = "1v75c0dqr0wp0dqd4hnci92qqs4hll8frqdbpswadgxm5chn91bw"; }) ]; urlPrefix = "mirror://ubuntu"; packages = commonDebPackages ++ [ "diffutils" "libc-bin" ]; }; - ubuntu1710x86_64 = { - name = "ubuntu-17.10-artful-amd64"; - fullName = "Ubuntu 17.10 Artful (amd64)"; + ubuntu1804x86_64 = { + name = "ubuntu-18.04-bionic-amd64"; + fullName = "Ubuntu 18.04 Bionic (amd64)"; packagesLists = [ (fetchurl { - url = "mirror://ubuntu/dists/artful/main/binary-amd64/Packages.xz"; - sha256 = "104g57j1l3vi8wb5f7rgjvjhf82ccs0vwhc59jfc4ynd51z7fqjk"; + url = "mirror://ubuntu/dists/bionic/main/binary-amd64/Packages.xz"; + sha256 = "1ls81bjyvmfz6i919kszl7xks1ibrh1xqhsk6698ackndkm0wp39"; }) (fetchurl { - url = "mirror://ubuntu/dists/artful/universe/binary-amd64/Packages.xz"; - sha256 = "1qzs95wfy9inaskfx9cf1l5yd3aaqwzy72zzi9xyvkxi75k5gcn4"; + url = "mirror://ubuntu/dists/bionic/universe/binary-amd64/Packages.xz"; + sha256 = "1832nqpn4ap95b3sj870xqayrza9in4kih9jkmjax27pq6x15v1r"; }) ]; urlPrefix = "mirror://ubuntu"; packages = commonDebPackages ++ [ "diffutils" "libc-bin" ]; }; - ubuntu1804i386 = { - name = "ubuntu-18.04-bionic-i386"; - fullName = "Ubuntu 18.04 Bionic (i386)"; + ubuntu2004i386 = { + name = "ubuntu-20.04-focal-i386"; + fullName = "Ubuntu 20.04 Focal (i386)"; packagesLists = [ (fetchurl { - url = "mirror://ubuntu/dists/bionic/main/binary-i386/Packages.xz"; - sha256 = "0f0v4131kwf7m7f8j3288rlqdxk1k3vqy74b7fcfd6jz9j8d840i"; + url = "mirror://ubuntu/dists/focal/main/binary-i386/Packages.xz"; + sha256 = "sha256-7RAYURoN3RKYQAHpwBS9TIV6vCmpURpphyMJQmV4wLc="; }) (fetchurl { - url = "mirror://ubuntu/dists/bionic/universe/binary-i386/Packages.xz"; - sha256 = "1v75c0dqr0wp0dqd4hnci92qqs4hll8frqdbpswadgxm5chn91bw"; + url = "mirror://ubuntu/dists/focal/universe/binary-i386/Packages.xz"; + sha256 = "sha256-oA551xVE80volUPgkMyvzpQ1d+GhuZd4DAe7dXZnULM="; }) ]; urlPrefix = "mirror://ubuntu"; packages = commonDebPackages ++ [ "diffutils" "libc-bin" ]; }; - ubuntu1804x86_64 = { - name = "ubuntu-18.04-bionic-amd64"; - fullName = "Ubuntu 18.04 Bionic (amd64)"; + ubuntu2004x86_64 = { + name = "ubuntu-20.04-focal-amd64"; + fullName = "Ubuntu 20.04 Focal (amd64)"; packagesLists = [ (fetchurl { - url = "mirror://ubuntu/dists/bionic/main/binary-amd64/Packages.xz"; - sha256 = "1ls81bjyvmfz6i919kszl7xks1ibrh1xqhsk6698ackndkm0wp39"; + url = "mirror://ubuntu/dists/focal/main/binary-amd64/Packages.xz"; + sha256 = "sha256-d1eSH/j+7Zw5NKDJk21EG6SiOL7j6myMHfXLzUP8mGE="; }) (fetchurl { - url = "mirror://ubuntu/dists/bionic/universe/binary-amd64/Packages.xz"; - sha256 = "1832nqpn4ap95b3sj870xqayrza9in4kih9jkmjax27pq6x15v1r"; + url = "mirror://ubuntu/dists/focal/universe/binary-amd64/Packages.xz"; + sha256 = "sha256-RqdG2seJvZU3rKVNsWgLnf9RwkgVMRE1A4IZnX2WudE="; }) ]; urlPrefix = "mirror://ubuntu"; packages = commonDebPackages ++ [ "diffutils" "libc-bin" ]; }; - debian8i386 = { - name = "debian-8.11-jessie-i386"; - fullName = "Debian 8.11 Jessie (i386)"; + debian9i386 = { + name = "debian-9.13-stretch-i386"; + fullName = "Debian 9.13 Stretch (i386)"; packagesList = fetchurl { - url = "mirror://debian/dists/jessie/main/binary-i386/Packages.xz"; - sha256 = "0adblarhx50yga900il6m25ng0csa81i3wid1dxxmydbdmri7v7d"; + url = "https://snapshot.debian.org/archive/debian/20210526T143040Z/dists/stretch/main/binary-i386/Packages.xz"; + sha256 = "sha256-fFRumd20wuVaYxzw0VPkAw5mQo8kIg+eXII15VSz9wA="; }; urlPrefix = "mirror://debian"; packages = commonDebianPackages; }; - debian8x86_64 = { - name = "debian-8.11-jessie-amd64"; - fullName = "Debian 8.11 Jessie (amd64)"; + debian9x86_64 = { + name = "debian-9.13-stretch-amd64"; + fullName = "Debian 9.13 Stretch (amd64)"; packagesList = fetchurl { - url = "mirror://debian/dists/jessie/main/binary-amd64/Packages.xz"; - sha256 = "09y1mv4kqllhxpk1ibjsyl5jig5bp0qxw6pp4sn56rglrpygmn5x"; + url = "https://snapshot.debian.org/archive/debian/20210526T143040Z/dists/stretch/main/binary-amd64/Packages.xz"; + sha256 = "sha256-1p4DEVpTGlBE3PtbQ90kYw4QNHkW0F4rna/Xz+ncMhw="; }; urlPrefix = "mirror://debian"; packages = commonDebianPackages; }; - debian9i386 = { - name = "debian-9.8-stretch-i386"; - fullName = "Debian 9.8 Stretch (i386)"; + debian10i386 = { + name = "debian-10.9-buster-i386"; + fullName = "Debian 10.9 Buster (i386)"; packagesList = fetchurl { - url = "http://snapshot.debian.org/archive/debian/20200301T030401Z/dists/stretch/main/binary-i386/Packages.xz"; - sha256 = "1jglr1d1jys3xddp8f7w9j05db39fah8xy4gfkpqbd1b5d2caslz"; + url = "https://snapshot.debian.org/archive/debian/20210526T143040Z/dists/buster/main/binary-i386/Packages.xz"; + sha256 = "sha256-zlkbKV+IGBCyWKD4v4LFM/EUA4TYS9fkLBPuF6MgUDo="; }; urlPrefix = "mirror://debian"; packages = commonDebianPackages; }; - debian9x86_64 = { - name = "debian-9.8-stretch-amd64"; - fullName = "Debian 9.8 Stretch (amd64)"; + debian10x86_64 = { + name = "debian-10.9-buster-amd64"; + fullName = "Debian 10.9 Buster (amd64)"; packagesList = fetchurl { - url = "http://snapshot.debian.org/archive/debian/20190503T090946Z/dists/stretch/main/binary-amd64/Packages.xz"; - sha256 = "01q00nl47p12n7wx0xclx59wf3zlkzrgj3zxpshyvb91xdnw5sh6"; + url = "https://snapshot.debian.org/archive/debian/20210526T143040Z/dists/buster/main/binary-amd64/Packages.xz"; + sha256 = "sha256-k13toY1b3CX7GBPQ7Jm24OMqCEsgPlGK8M99x57o69o="; }; urlPrefix = "mirror://debian"; packages = commonDebianPackages; }; - - }; @@ -1167,7 +1127,7 @@ rec { "passwd" ]; - commonDebianPackages = commonDebPackages ++ [ "sysvinit" "diff" "mktemp" ]; + commonDebianPackages = commonDebPackages ++ [ "sysvinit" "diff" ]; /* A set of functions that build the Linux distributions specified @@ -1196,4 +1156,4 @@ rec { `debDistros' sets. */ diskImages = lib.mapAttrs (name: f: f {}) diskImageFuns; -} // import ./windows pkgs +} diff --git a/pkgs/build-support/vm/windows/bootstrap.nix b/pkgs/build-support/vm/windows/bootstrap.nix deleted file mode 100644 index 3b06d8f4749..00000000000 --- a/pkgs/build-support/vm/windows/bootstrap.nix +++ /dev/null @@ -1,83 +0,0 @@ -{ stdenv, fetchurl, vmTools, writeScript, writeText, runCommand, makeInitrd -, python, perl, coreutils, dosfstools, gzip, mtools, netcat-gnu, openssh, qemu -, samba, socat, vde2, cdrkit, pathsFromGraph, gnugrep -}: - -{ isoFile, productKey, arch ? null }: - -with stdenv.lib; - -let - controller = import ./controller { - inherit stdenv writeScript vmTools makeInitrd; - inherit samba vde2 openssh socat netcat-gnu coreutils gzip gnugrep; - }; - - mkCygwinImage = import ./cygwin-iso { - inherit stdenv fetchurl runCommand python perl cdrkit pathsFromGraph; - arch = let - defaultArch = if stdenv.is64bit then "x86_64" else "i686"; - in if arch == null then defaultArch else arch; - }; - - installer = import ./install { - inherit controller mkCygwinImage; - inherit stdenv runCommand openssh qemu writeText dosfstools mtools; - }; -in rec { - installedVM = installer { - inherit isoFile productKey; - }; - - runInVM = img: attrs: controller (attrs // { - inherit (installedVM) sshKey; - qemuArgs = attrs.qemuArgs or [] ++ [ - "-boot order=c" - "-drive file=${img},index=0,media=disk" - ]; - }); - - runAndSuspend = let - drives = { - s = { - source = "nixstore"; - target = "/nix/store"; - }; - x = { - source = "xchg"; - target = "/tmp/xchg"; - }; - }; - - genDriveCmds = letter: { source, target }: [ - "net use ${letter}: '\\\\192.168.0.2\\${source}' /persistent:yes" - "mkdir -p '${target}'" - "mount -o bind '/cygdrive/${letter}' '${target}'" - "echo '/cygdrive/${letter} ${target} none bind 0 0' >> /etc/fstab" - ]; - in runInVM "winvm.img" { - command = concatStringsSep " && " ([ - "net config server /autodisconnect:-1" - ] ++ concatLists (mapAttrsToList genDriveCmds drives)); - suspendTo = "state.gz"; - }; - - suspendedVM = stdenv.mkDerivation { - name = "cygwin-suspended-vm"; - buildCommand = '' - ${qemu}/bin/qemu-img create \ - -b "${installedVM}/disk.img" \ - -f qcow2 winvm.img - ${runAndSuspend} - mkdir -p "$out" - cp winvm.img "$out/disk.img" - cp state.gz "$out/state.gz" - ''; - }; - - resumeAndRun = command: runInVM "${suspendedVM}/disk.img" { - resumeFrom = "${suspendedVM}/state.gz"; - qemuArgs = singleton "-snapshot"; - inherit command; - }; -} diff --git a/pkgs/build-support/vm/windows/controller/default.nix b/pkgs/build-support/vm/windows/controller/default.nix deleted file mode 100644 index 9d13983a283..00000000000 --- a/pkgs/build-support/vm/windows/controller/default.nix +++ /dev/null @@ -1,263 +0,0 @@ -{ stdenv, writeScript, vmTools, makeInitrd -, samba, vde2, openssh, socat, netcat-gnu, coreutils, gnugrep, gzip -, runtimeShell -}: - -{ sshKey -, qemuArgs ? [] -, command ? "sync" -, suspendTo ? null -, resumeFrom ? null -, installMode ? false -}: - -with stdenv.lib; - -let - preInitScript = writeScript "preinit.sh" '' - #!${vmTools.initrdUtils}/bin/ash -e - export PATH=${vmTools.initrdUtils}/bin - mount -t proc none /proc - mount -t sysfs none /sys - for arg in $(cat /proc/cmdline); do - if [ "x''${arg#command=}" != "x$arg" ]; then - command="''${arg#command=}" - fi - done - - for i in $(cat ${modulesClosure}/insmod-list); do - insmod $i - done - - mkdir -p /dev /fs - - mount -t tmpfs none /dev - mknod /dev/null c 1 3 - mknod /dev/zero c 1 5 - mknod /dev/random c 1 8 - mknod /dev/urandom c 1 9 - mknod /dev/tty c 5 0 - - ifconfig lo up - ifconfig eth0 up 192.168.0.2 - - mount -t tmpfs none /fs - mkdir -p /fs/nix/store /fs/xchg /fs/dev /fs/sys /fs/proc /fs/etc /fs/tmp - - mount -o bind /dev /fs/dev - mount -t sysfs none /fs/sys - mount -t proc none /fs/proc - - mount -t 9p \ - -o trans=virtio,version=9p2000.L,cache=loose \ - store /fs/nix/store - - mount -t 9p \ - -o trans=virtio,version=9p2000.L \ - xchg /fs/xchg - - echo root:x:0:0::/root:/bin/false > /fs/etc/passwd - - set +e - chroot /fs $command $out - echo $? > /fs/xchg/in-vm-exit - - poweroff -f - ''; - - initrd = makeInitrd { - contents = singleton { - object = preInitScript; - symlink = "/init"; - }; - }; - - loopForever = "while :; do ${coreutils}/bin/sleep 1; done"; - - initScript = writeScript "init.sh" ('' - #!${runtimeShell} - ${coreutils}/bin/cp -L "${sshKey}" /ssh.key - ${coreutils}/bin/chmod 600 /ssh.key - '' + (if installMode then '' - echo -n "Waiting for Windows installation to finish..." - while ! ${netcat-gnu}/bin/netcat -z 192.168.0.1 22; do - echo -n . - # Print a dot every 10 seconds only to shorten line length. - ${coreutils}/bin/sleep 10 - done - ${coreutils}/bin/touch /xchg/waiting_done - echo " success." - # Loop forever, because this VM is going to be killed. - ${loopForever} - '' else '' - ${coreutils}/bin/mkdir -p /etc/samba /etc/samba/private \ - /var/lib/samba /var/log /var/run - ${coreutils}/bin/cat > /etc/samba/smb.conf <<CONFIG - [global] - security = user - map to guest = Bad User - guest account = root - workgroup = cygwin - netbios name = controller - server string = %h - log level = 1 - max log size = 1000 - log file = /var/log/samba.log - - [nixstore] - path = /nix/store - writable = yes - guest ok = yes - - [xchg] - path = /xchg - writable = yes - guest ok = yes - CONFIG - - ${samba}/sbin/nmbd -D - ${samba}/sbin/smbd -D - - echo -n "Waiting for Windows VM to become available..." - while ! ${netcat-gnu}/bin/netcat -z 192.168.0.1 22; do - echo -n . - ${coreutils}/bin/sleep 1 - done - ${coreutils}/bin/touch /xchg/waiting_done - echo " success." - - ${openssh}/bin/ssh \ - -o UserKnownHostsFile=/dev/null \ - -o StrictHostKeyChecking=no \ - -i /ssh.key \ - -l Administrator \ - 192.168.0.1 -- ${lib.escapeShellArg command} - '') + optionalString (suspendTo != null) '' - ${coreutils}/bin/touch /xchg/suspend_now - ${loopForever} - ''); - - kernelAppend = concatStringsSep " " [ - "panic=1" - "loglevel=4" - "console=tty1" - "console=ttyS0" - "command=${initScript}" - ]; - - controllerQemuArgs = concatStringsSep " " (maybeKvm64 ++ [ - "-pidfile $CTRLVM_PIDFILE" - "-nographic" - "-no-reboot" - "-virtfs local,path=/nix/store,security_model=none,mount_tag=store" - "-virtfs local,path=$XCHG_DIR,security_model=none,mount_tag=xchg" - "-kernel ${modulesClosure.kernel}/bzImage" - "-initrd ${initrd}/initrd" - "-append \"${kernelAppend}\"" - "-net nic,vlan=0,macaddr=52:54:00:12:01:02,model=virtio" - "-net vde,vlan=0,sock=$QEMU_VDE_SOCKET" - ]); - - maybeKvm64 = optional (stdenv.hostPlatform.system == "x86_64-linux") "-cpu kvm64"; - - cygwinQemuArgs = concatStringsSep " " (maybeKvm64 ++ [ - "-monitor unix:$MONITOR_SOCKET,server,nowait" - "-pidfile $WINVM_PIDFILE" - "-nographic" - "-net nic,vlan=0,macaddr=52:54:00:12:01:01" - "-net vde,vlan=0,sock=$QEMU_VDE_SOCKET" - "-rtc base=2010-01-01,clock=vm" - ] ++ qemuArgs ++ optionals (resumeFrom != null) [ - "-incoming 'exec: ${gzip}/bin/gzip -c -d \"${resumeFrom}\"'" - ]); - - modulesClosure = overrideDerivation vmTools.modulesClosure (o: { - rootModules = o.rootModules ++ singleton "virtio_net"; - }); - - preVM = '' - (set; declare -p) > saved-env - XCHG_DIR="$(${coreutils}/bin/mktemp -d nix-vm.XXXXXXXXXX --tmpdir)" - ${coreutils}/bin/mv saved-env "$XCHG_DIR/" - - eval "$preVM" - - QEMU_VDE_SOCKET="$(pwd)/vde.ctl" - MONITOR_SOCKET="$(pwd)/monitor" - WINVM_PIDFILE="$(pwd)/winvm.pid" - CTRLVM_PIDFILE="$(pwd)/ctrlvm.pid" - ${vde2}/bin/vde_switch -s "$QEMU_VDE_SOCKET" --dirmode 0700 & - echo 'alive?' | ${socat}/bin/socat - \ - UNIX-CONNECT:$QEMU_VDE_SOCKET/ctl,retry=20 - ''; - - vmExec = '' - ${vmTools.qemuProg} ${controllerQemuArgs} & - ${vmTools.qemuProg} ${cygwinQemuArgs} & - echo -n "Waiting for VMs to start up..." - timeout=60 - while ! test -e "$WINVM_PIDFILE" -a -e "$CTRLVM_PIDFILE"; do - timeout=$(($timeout - 1)) - echo -n . - if test $timeout -le 0; then - echo " timed out." - exit 1 - fi - ${coreutils}/bin/sleep 1 - done - echo " done." - ''; - - checkDropOut = '' - if ! test -e "$XCHG_DIR/waiting_done" && - ! kill -0 $(< "$WINVM_PIDFILE"); then - echo "Windows VM has dropped out early, bailing out!" >&2 - exit 1 - fi - ''; - - toMonitor = "${socat}/bin/socat - UNIX-CONNECT:$MONITOR_SOCKET"; - - postVM = if suspendTo != null then '' - while ! test -e "$XCHG_DIR/suspend_now"; do - ${checkDropOut} - ${coreutils}/bin/sleep 1 - done - ${toMonitor} <<CMD - stop - migrate_set_speed 4095m - migrate "exec:${gzip}/bin/gzip -c > '${suspendTo}'" - CMD - echo -n "Waiting for memory dump to finish..." - while ! echo info migrate | ${toMonitor} | \ - ${gnugrep}/bin/grep -qi '^migration *status: *complete'; do - ${coreutils}/bin/sleep 1 - echo -n . - done - echo " done." - echo quit | ${toMonitor} - wait $(< "$WINVM_PIDFILE") - eval "$postVM" - exit 0 - '' else if installMode then '' - wait $(< "$WINVM_PIDFILE") - eval "$postVM" - exit 0 - '' else '' - while kill -0 $(< "$CTRLVM_PIDFILE"); do - ${checkDropOut} - done - if ! test -e "$XCHG_DIR/in-vm-exit"; then - echo "Virtual machine didn't produce an exit code." - exit 1 - fi - eval "$postVM" - exit $(< "$XCHG_DIR/in-vm-exit") - ''; - -in writeScript "run-cygwin-vm.sh" '' - #!${stdenv.shell} -e - ${preVM} - ${vmExec} - ${postVM} -'' diff --git a/pkgs/build-support/vm/windows/cygwin-iso/default.nix b/pkgs/build-support/vm/windows/cygwin-iso/default.nix deleted file mode 100644 index 76cd41a75bc..00000000000 --- a/pkgs/build-support/vm/windows/cygwin-iso/default.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ stdenv, fetchurl, runCommand, python, perl, xorriso, pathsFromGraph -, arch ? "x86_64" -}: - -{ packages ? [] -, mirror ? "http://ftp.gwdg.de/pub/linux/sources.redhat.com/cygwin" -, extraContents ? [] -}: - -let - cygPkgList = if arch == "x86_64" then fetchurl { - url = "${mirror}/x86_64/setup.ini"; - sha256 = "0arrxvxbl85l82iy648snx5cl952w791p45p0dfg1xpiaf96cbkj"; - } else fetchurl { - url = "${mirror}/x86/setup.ini"; - sha256 = "1fayx34868vd5h2nah7chiw65sl3i9qzrwvs7lrlv2h8k412vb69"; - }; - - cygwinCross = (import ../../../../.. { - localSystem = stdenv.hostPlatform; - crossSystem = { - libc = "msvcrt"; - platform = {}; - inherit arch; - config = "${arch}-w64-mingw32"; - }; - }).windows.cygwinSetup; - - makeCygwinClosure = { packages, packageList }: let - expr = import (runCommand "cygwin.nix" { buildInputs = [ python ]; } '' - python ${./mkclosure.py} "${packages}" ${toString packageList} > "$out" - ''); - gen = { url, hash }: { - source = fetchurl { - url = "${mirror}/${url}"; - sha512 = hash; - }; - target = url; - }; - in map gen expr; - -in import ../../../../../nixos/lib/make-iso9660-image.nix { - inherit stdenv perl xorriso pathsFromGraph; - syslinux = null; - contents = [ - { source = "${cygwinCross}/bin/setup.exe"; - target = "setup.exe"; - } - { source = cygPkgList; - target = "setup.ini"; - } - ] ++ makeCygwinClosure { - packages = cygPkgList; - packageList = packages; - } ++ extraContents; -} diff --git a/pkgs/build-support/vm/windows/cygwin-iso/mkclosure.py b/pkgs/build-support/vm/windows/cygwin-iso/mkclosure.py deleted file mode 100644 index 4c0d67c43ba..00000000000 --- a/pkgs/build-support/vm/windows/cygwin-iso/mkclosure.py +++ /dev/null @@ -1,78 +0,0 @@ -# Ugliest Python code I've ever written. -- aszlig -import sys - -def get_plist(path): - in_pack = False - in_str = False - current_key = None - buf = "" - packages = {} - package_name = None - package_attrs = {} - with open(path, 'r') as setup: - for line in setup: - if in_str and line.rstrip().endswith('"'): - package_attrs[current_key] = buf + line.rstrip()[:-1] - in_str = False - continue - elif in_str: - buf += line - continue - - if line.startswith('@'): - in_pack = True - package_name = line[1:].strip() - package_attrs = {} - elif in_pack and ':' in line: - key, value = line.split(':', 1) - if value.lstrip().startswith('"'): - if value.lstrip()[1:].rstrip().endswith('"'): - value = value.strip().strip('"') - else: - in_str = True - current_key = key.strip().lower() - buf = value.lstrip()[1:] - continue - package_attrs[key.strip().lower()] = value.strip() - elif in_pack: - in_pack = False - packages[package_name] = package_attrs - return packages - -def main(): - packages = get_plist(sys.argv[1]) - to_include = set() - - def traverse(package): - to_include.add(package) - attrs = packages.get(package, {}) - deps = attrs.get('requires', '').split() - for new_dep in set(deps) - to_include: - traverse(new_dep) - - map(traverse, sys.argv[2:]) - - sys.stdout.write('[\n') - for package, attrs in packages.iteritems(): - if package not in to_include: - cats = [c.lower() for c in attrs.get('category', '').split()] - if 'base' not in cats: - continue - - install_line = attrs.get('install') - if install_line is None: - continue - - url, size, hash = install_line.split(' ', 2) - - pack = [ - ' {', - ' url = "{0}";'.format(url), - ' hash = "{0}";'.format(hash), - ' }', - ]; - sys.stdout.write('\n'.join(pack) + '\n') - sys.stdout.write(']\n') - -if __name__ == '__main__': - main() diff --git a/pkgs/build-support/vm/windows/default.nix b/pkgs/build-support/vm/windows/default.nix deleted file mode 100644 index 309241c36de..00000000000 --- a/pkgs/build-support/vm/windows/default.nix +++ /dev/null @@ -1,44 +0,0 @@ -#note: the hardcoded /bin/sh is required for the VM's cygwin shell -pkgs: - -let - bootstrapper = import ./bootstrap.nix { - inherit (pkgs) stdenv vmTools writeScript writeText runCommand makeInitrd; - inherit (pkgs) coreutils dosfstools gzip mtools netcat-gnu openssh qemu samba; - inherit (pkgs) socat vde2 fetchurl python perl cdrkit pathsFromGraph; - inherit (pkgs) gnugrep; - }; - - builder = '' - source /tmp/xchg/saved-env 2> /dev/null || true - export NIX_STORE=/nix/store - export NIX_BUILD_TOP=/tmp - export TMPDIR=/tmp - export PATH=/empty - cd "$NIX_BUILD_TOP" - exec $origBuilder $origArgs - ''; - -in { - runInWindowsVM = drv: pkgs.lib.overrideDerivation drv (attrs: let - bootstrap = bootstrapper attrs.windowsImage; - in { - requiredSystemFeatures = [ "kvm" ]; - builder = pkgs.stdenv.shell; - args = ["-e" (bootstrap.resumeAndRun builder)]; - windowsImage = bootstrap.suspendedVM; - origArgs = attrs.args; - origBuilder = if attrs.builder == attrs.stdenv.shell - then "/bin/sh" - else attrs.builder; - - postHook = '' - PATH=/usr/bin:/bin:/usr/sbin:/sbin - SHELL=/bin/sh - eval "$origPostHook" - ''; - - origPostHook = attrs.postHook or ""; - fixupPhase = ":"; - }); -} diff --git a/pkgs/build-support/vm/windows/install/default.nix b/pkgs/build-support/vm/windows/install/default.nix deleted file mode 100644 index fe8e8f61de0..00000000000 --- a/pkgs/build-support/vm/windows/install/default.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ stdenv, runCommand, openssh, qemu, controller, mkCygwinImage -, writeText, dosfstools, mtools -}: - -{ isoFile -, productKey -}: - -let - bootstrapAfterLogin = runCommand "bootstrap.sh" {} '' - cat > "$out" <<EOF - mkdir -p ~/.ssh - cat > ~/.ssh/authorized_keys <<PUBKEY - $(cat "${cygwinSshKey}/key.pub") - PUBKEY - ssh-host-config -y -c 'binmode ntsec' -w dummy - cygrunsrv -S sshd - shutdown -s 5 - EOF - ''; - - cygwinSshKey = stdenv.mkDerivation { - name = "snakeoil-ssh-cygwin"; - buildCommand = '' - mkdir -p "$out" - ${openssh}/bin/ssh-keygen -t ecdsa -f "$out/key" -N "" - ''; - }; - - sshKey = "${cygwinSshKey}/key"; - - packages = [ "openssh" "shutdown" ]; - - floppyCreator = import ./unattended-image.nix { - inherit stdenv writeText dosfstools mtools; - }; - - instfloppy = floppyCreator { - cygwinPackages = packages; - inherit productKey; - }; - - cygiso = mkCygwinImage { - inherit packages; - extraContents = stdenv.lib.singleton { - source = bootstrapAfterLogin; - target = "bootstrap.sh"; - }; - }; - - installController = controller { - inherit sshKey; - installMode = true; - qemuArgs = [ - "-boot order=c,once=d" - "-drive file=${instfloppy},readonly,index=0,if=floppy" - "-drive file=winvm.img,index=0,media=disk" - "-drive file=${isoFile},index=1,media=cdrom" - "-drive file=${cygiso}/iso/cd.iso,index=2,media=cdrom" - ]; - }; - -in stdenv.mkDerivation { - name = "cygwin-base-vm"; - buildCommand = '' - ${qemu}/bin/qemu-img create -f qcow2 winvm.img 2G - ${installController} - mkdir -p "$out" - cp winvm.img "$out/disk.img" - ''; - passthru = { - inherit sshKey; - }; -} diff --git a/pkgs/build-support/vm/windows/install/unattended-image.nix b/pkgs/build-support/vm/windows/install/unattended-image.nix deleted file mode 100644 index 5b1ff84cf44..00000000000 --- a/pkgs/build-support/vm/windows/install/unattended-image.nix +++ /dev/null @@ -1,123 +0,0 @@ -{ stdenv, writeText, dosfstools, mtools }: - -{ productKey -, shExecAfterwards ? "E:\\bootstrap.sh" -, cygwinRoot ? "C:\\cygwin" -, cygwinSetup ? "E:\\setup.exe" -, cygwinRepository ? "E:\\" -, cygwinPackages ? [ "openssh" ] -}: - -let - afterSetup = [ - cygwinSetup - "-L -n -q" - "-l ${cygwinRepository}" - "-R ${cygwinRoot}" - "-C base" - ] ++ map (p: "-P ${p}") cygwinPackages; - - winXpUnattended = writeText "winnt.sif" '' - [Data] - AutoPartition = 1 - AutomaticUpdates = 0 - MsDosInitiated = 0 - UnattendedInstall = Yes - - [Unattended] - DUDisable = Yes - DriverSigningPolicy = Ignore - Hibernation = No - OemPreinstall = No - OemSkipEula = Yes - Repartition = Yes - TargetPath = \WINDOWS - UnattendMode = FullUnattended - UnattendSwitch = Yes - WaitForReboot = No - - [GuiUnattended] - AdminPassword = "nopasswd" - AutoLogon = Yes - AutoLogonCount = 1 - OEMSkipRegional = 1 - OemSkipWelcome = 1 - ServerWelcome = No - TimeZone = 85 - - [UserData] - ComputerName = "cygwin" - FullName = "cygwin" - OrgName = "" - ProductKey = "${productKey}" - - [Networking] - InstallDefaultComponents = Yes - - [Identification] - JoinWorkgroup = cygwin - - [NetAdapters] - PrimaryAdapter = params.PrimaryAdapter - - [params.PrimaryAdapter] - InfID = * - - [params.MS_MSClient] - - [NetProtocols] - MS_TCPIP = params.MS_TCPIP - - [params.MS_TCPIP] - AdapterSections=params.MS_TCPIP.PrimaryAdapter - - [params.MS_TCPIP.PrimaryAdapter] - DHCP = No - IPAddress = 192.168.0.1 - SpecificTo = PrimaryAdapter - SubnetMask = 255.255.255.0 - WINS = No - - ; Turn off all components - [Components] - ${stdenv.lib.concatMapStrings (comp: "${comp} = Off\n") [ - "AccessOpt" "Appsrv_console" "Aspnet" "BitsServerExtensionsISAPI" - "BitsServerExtensionsManager" "Calc" "Certsrv" "Certsrv_client" - "Certsrv_server" "Charmap" "Chat" "Clipbook" "Cluster" "Complusnetwork" - "Deskpaper" "Dialer" "Dtcnetwork" "Fax" "Fp_extensions" "Fp_vdir_deploy" - "Freecell" "Hearts" "Hypertrm" "IEAccess" "IEHardenAdmin" "IEHardenUser" - "Iis_asp" "Iis_common" "Iis_ftp" "Iis_inetmgr" "Iis_internetdataconnector" - "Iis_nntp" "Iis_serversideincludes" "Iis_smtp" "Iis_webdav" "Iis_www" - "Indexsrv_system" "Inetprint" "Licenseserver" "Media_clips" "Media_utopia" - "Minesweeper" "Mousepoint" "Msmq_ADIntegrated" "Msmq_Core" - "Msmq_HTTPSupport" "Msmq_LocalStorage" "Msmq_MQDSService" - "Msmq_RoutingSupport" "Msmq_TriggersService" "Msnexplr" "Mswordpad" - "Netcis" "Netoc" "OEAccess" "Objectpkg" "Paint" "Pinball" "Pop3Admin" - "Pop3Service" "Pop3Srv" "Rec" "Reminst" "Rootautoupdate" "Rstorage" "SCW" - "Sakit_web" "Solitaire" "Spider" "TSWebClient" "Templates" - "TerminalServer" "UDDIAdmin" "UDDIDatabase" "UDDIWeb" "Vol" "WMAccess" - "WMPOCM" "WbemMSI" "Wms" "Wms_admin_asp" "Wms_admin_mmc" "Wms_isapi" - "Wms_server" "Zonegames" - ]} - - [WindowsFirewall] - Profiles = WindowsFirewall.TurnOffFirewall - - [WindowsFirewall.TurnOffFirewall] - Mode = 0 - - [SetupParams] - UserExecute = "${stdenv.lib.concatStringsSep " " afterSetup}" - - [GuiRunOnce] - Command0 = "${cygwinRoot}\bin\bash -l ${shExecAfterwards}" - ''; - -in stdenv.mkDerivation { - name = "unattended-floppy.img"; - buildCommand = '' - dd if=/dev/zero of="$out" count=1440 bs=1024 - ${dosfstools}/sbin/mkfs.msdos "$out" - ${mtools}/bin/mcopy -i "$out" "${winXpUnattended}" ::winnt.sif - ''; -} diff --git a/pkgs/build-support/wrapper-common/utils.bash b/pkgs/build-support/wrapper-common/utils.bash index 8c4680a8e44..cb3552ebc54 100644 --- a/pkgs/build-support/wrapper-common/utils.bash +++ b/pkgs/build-support/wrapper-common/utils.bash @@ -13,7 +13,9 @@ accumulateRoles() { fi } -mangleVarList() { +mangleVarListGeneric() { + local sep="$1" + shift local var="$1" shift local -a role_suffixes=("$@") @@ -25,11 +27,15 @@ mangleVarList() { for suffix in "${role_suffixes[@]}"; do local inputVar="${var}${suffix}" if [ -v "$inputVar" ]; then - export ${outputVar}+="${!outputVar:+ }${!inputVar}" + export ${outputVar}+="${!outputVar:+$sep}${!inputVar}" fi done } +mangleVarList() { + mangleVarListGeneric " " "$@" +} + mangleVarBool() { local var="$1" shift @@ -49,6 +55,35 @@ mangleVarBool() { done } +# Combine a singular value from all roles. If multiple roles are being served, +# and the value differs in these roles then the request is impossible to +# satisfy and we abort immediately. +mangleVarSingle() { + local var="$1" + shift + local -a role_suffixes=("$@") + + local outputVar="${var}_@suffixSalt@" + for suffix in "${role_suffixes[@]}"; do + local inputVar="${var}${suffix}" + if [ -v "$inputVar" ]; then + if [ -v "$outputVar" ]; then + if [ "${!outputVar}" != "${!inputVar}" ]; then + { + echo "Multiple conflicting values defined for $outputVar" + echo "Existing value is ${!outputVar}" + echo "Attempting to set to ${!inputVar} via $inputVar" + } >&2 + + exit 1 + fi + else + declare -gx ${outputVar}="${!inputVar}" + fi + fi + done +} + skip () { if (( "${NIX_DEBUG:-0}" >= 1 )); then echo "skipping impure path $1" >&2 @@ -69,9 +104,13 @@ badPath() { # directory (including the build directory). test \ "$p" != "/dev/null" -a \ - "${p:0:${#NIX_STORE}}" != "$NIX_STORE" -a \ - "${p:0:4}" != "/tmp" -a \ - "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP" + "${p#${NIX_STORE}}" = "$p" -a \ + "${p#${NIX_BUILD_TOP}}" = "$p" -a \ + "${p#/tmp}" = "$p" -a \ + "${p#${TMP:-/tmp}}" = "$p" -a \ + "${p#${TMPDIR:-/tmp}}" = "$p" -a \ + "${p#${TEMP:-/tmp}}" = "$p" -a \ + "${p#${TEMPDIR:-/tmp}}" = "$p" } expandResponseParams() { @@ -90,3 +129,38 @@ expandResponseParams() { fi done } + +checkLinkType() { + local arg mode + type="dynamic" + for arg in "$@"; do + if [[ "$arg" = -static ]]; then + type="static" + elif [[ "$arg" = -static-pie ]]; then + type="static-pie" + fi + done + echo "$type" +} + +# When building static-pie executables we cannot have rpath +# set. At least glibc requires rpath to be empty +filterRpathFlags() { + local linkType=$1 ret="" i + shift + + if [[ "$linkType" == "static-pie" ]]; then + while [[ "$#" -gt 0 ]]; do + i="$1"; shift 1 + if [[ "$i" == -rpath ]]; then + # also skip its argument + shift + else + ret+="$i " + fi + done + else + ret=$@ + fi + echo $ret +} diff --git a/pkgs/build-support/writers/default.nix b/pkgs/build-support/writers/default.nix index 4673b4e6cd8..47919c251af 100644 --- a/pkgs/build-support/writers/default.nix +++ b/pkgs/build-support/writers/default.nix @@ -1,4 +1,4 @@ -{ pkgs, lib }: +{ pkgs, lib, gawk, gnused, gixy }: with lib; rec { @@ -63,7 +63,7 @@ rec { # # Examples: # writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; } - makeBinWriter = { compileScript }: nameOrPath: content: + makeBinWriter = { compileScript, strip ? true }: nameOrPath: content: assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); assert lib.or (types.path.check content) (types.str.check content); let @@ -76,6 +76,8 @@ rec { contentPath = content; }) '' ${compileScript} + ${lib.optionalString strip + "${pkgs.binutils-unwrapped}/bin/strip --strip-unneeded $out"} ${optionalString (types.path.check nameOrPath) '' mv $out tmp mkdir -p $out/$(dirname "${nameOrPath}") @@ -109,7 +111,10 @@ rec { # return 0; # } # '' - writeC = name: { libraries ? [] }: + writeC = name: { + libraries ? [], + strip ? true + }: makeBinWriter { compileScript = '' PATH=${makeBinPath [ @@ -117,7 +122,7 @@ rec { pkgs.coreutils pkgs.findutils pkgs.gcc - pkgs.pkgconfig + pkgs.pkg-config ]} export PKG_CONFIG_PATH=${concatMapStringsSep ":" (pkg: "${pkg}/lib/pkgconfig") libraries} gcc \ @@ -131,8 +136,8 @@ rec { -Wall \ -x c \ "$contentPath" - strip --strip-unneeded "$out" ''; + inherit strip; } name; # writeCBin takes the same arguments as writeC but outputs a directory (like writeScriptBin) @@ -164,21 +169,39 @@ rec { # ''; writeHaskell = name: { libraries ? [], - ghc ? pkgs.ghc + ghc ? pkgs.ghc, + ghcArgs ? [], + strip ? true }: makeBinWriter { compileScript = '' cp $contentPath tmp.hs - ${ghc.withPackages (_: libraries )}/bin/ghc tmp.hs + ${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs} tmp.hs mv tmp $out - ${pkgs.binutils-unwrapped}/bin/strip --strip-unneeded "$out" ''; + inherit strip; } name; # writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin) writeHaskellBin = name: writeHaskell "/bin/${name}"; + writeRust = name: { + rustc ? pkgs.rustc, + rustcArgs ? [], + strip ? true + }: + makeBinWriter { + compileScript = '' + cp "$contentPath" tmp.rs + PATH=${makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} -o "$out" tmp.rs + ''; + inherit strip; + } name; + + writeRustBin = name: + writeRust "/bin/${name}"; + # writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and # returns an executable # @@ -218,10 +241,11 @@ rec { writeNginxConfig = name: text: pkgs.runCommandLocal name { inherit text; passAsFile = [ "text" ]; + nativeBuildInputs = [ gawk gnused gixy ]; } /* sh */ '' # nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16 - ${pkgs.gawk}/bin/awk -f ${awkFormatNginx} "$textPath" | ${pkgs.gnused}/bin/sed '/^\s*$/d' > $out - ${pkgs.gixy}/bin/gixy $out + awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out + gixy $out ''; # writePerl takes a name an attributeset with libraries and some perl sourcecode and @@ -233,18 +257,9 @@ rec { # print "Howdy!\n" if true; # '' writePerl = name: { libraries ? [] }: - let - perl-env = pkgs.buildEnv { - name = "perl-environment"; - paths = libraries; - pathsToLink = [ - "/${pkgs.perl.libPrefix}" - ]; - }; - in - makeScriptWriter { - interpreter = "${pkgs.perl}/bin/perl -I ${perl-env}/${pkgs.perl.libPrefix}"; - } name; + makeScriptWriter { + interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl"; + } name; # writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin) writePerlBin = name: diff --git a/pkgs/build-support/writers/test.nix b/pkgs/build-support/writers/test.nix index d284bda43d0..00cad9a96b5 100644 --- a/pkgs/build-support/writers/test.nix +++ b/pkgs/build-support/writers/test.nix @@ -1,25 +1,23 @@ -{ - glib, - haskellPackages, - lib, - nodePackages, - perlPackages, - python2Packages, - python3Packages, - runCommand, - stdenv, - writers, - writeText +{ glib +, haskellPackages +, lib +, nodePackages +, perlPackages +, python2Packages +, python3Packages +, runCommand +, writers +, writeText }: with writers; let bin = { - bash = writeBashBin "test_writers" '' + bash = writeBashBin "test-writers-bash-bin" '' if [[ "test" == "test" ]]; then echo "success"; fi ''; - c = writeCBin "test_writers" { libraries = [ ]; } '' + c = writeCBin "test-writers-c" { libraries = [ ]; } '' #include <stdio.h> int main() { printf("success\n"); @@ -27,11 +25,17 @@ let } ''; - dash = writeDashBin "test_writers" '' + dash = writeDashBin "test-writers-dash-bin" '' test '~' = '~' && echo 'success' ''; - haskell = writeHaskellBin "test_writers" { libraries = [ haskellPackages.acme-default ]; } '' + rust = writeRustBin "test-writers-rust-bin" {} '' + fn main(){ + println!("success") + } + ''; + + haskell = writeHaskellBin "test-writers-haskell-bin" { libraries = [ haskellPackages.acme-default ]; } '' import Data.Default int :: Int @@ -43,7 +47,7 @@ let _ -> print "fail" ''; - js = writeJSBin "test_writers" { libraries = [ nodePackages.semver ]; } '' + js = writeJSBin "test-writers-js-bin" { libraries = [ nodePackages.semver ]; } '' var semver = require('semver'); if (semver.valid('1.2.3')) { @@ -53,12 +57,12 @@ let } ''; - perl = writePerlBin "test_writers" { libraries = [ perlPackages.boolean ]; } '' + perl = writePerlBin "test-writers-perl-bin" { libraries = [ perlPackages.boolean ]; } '' use boolean; print "success\n" if true; ''; - python2 = writePython2Bin "test_writers" { libraries = [ python2Packages.enum ]; } '' + python2 = writePython2Bin "test-writers-python2-bin" { libraries = [ python2Packages.enum ]; } '' from enum import Enum @@ -69,7 +73,7 @@ let print Test.a ''; - python3 = writePython3Bin "test_writers" { libraries = [ python3Packages.pyyaml ]; } '' + python3 = writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } '' import yaml y = yaml.load(""" @@ -80,11 +84,11 @@ let }; simple = { - bash = writeBash "test_bash" '' + bash = writeBash "test-writers-bash" '' if [[ "test" == "test" ]]; then echo "success"; fi ''; - c = writeC "test_c" { libraries = [ glib.dev ]; } '' + c = writeC "test-writers-c" { libraries = [ glib.dev ]; } '' #include <gio/gio.h> #include <stdio.h> int main() { @@ -102,11 +106,11 @@ let } ''; - dash = writeDash "test_dash" '' + dash = writeDash "test-writers-dash" '' test '~' = '~' && echo 'success' ''; - haskell = writeHaskell "test_haskell" { libraries = [ haskellPackages.acme-default ]; } '' + haskell = writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } '' import Data.Default int :: Int @@ -118,7 +122,7 @@ let _ -> print "fail" ''; - js = writeJS "test_js" { libraries = [ nodePackages.semver ]; } '' + js = writeJS "test-writers-js" { libraries = [ nodePackages.semver ]; } '' var semver = require('semver'); if (semver.valid('1.2.3')) { @@ -128,12 +132,12 @@ let } ''; - perl = writePerl "test_perl" { libraries = [ perlPackages.boolean ]; } '' + perl = writePerl "test-writers-perl" { libraries = [ perlPackages.boolean ]; } '' use boolean; print "success\n" if true; ''; - python2 = writePython2 "test_python2" { libraries = [ python2Packages.enum ]; } '' + python2 = writePython2 "test-writers-python2" { libraries = [ python2Packages.enum ]; } '' from enum import Enum @@ -144,7 +148,7 @@ let print Test.a ''; - python3 = writePython3 "test_python3" { libraries = [ python3Packages.pyyaml ]; } '' + python3 = writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } '' import yaml y = yaml.load(""" @@ -153,21 +157,21 @@ let print(y[0]['test']) ''; - python2NoLibs = writePython2 "test_python2_no_libs" {} '' + python2NoLibs = writePython2 "test-writers-python2-no-libs" {} '' print("success") ''; - python3NoLibs = writePython3 "test_python3_no_libs" {} '' + python3NoLibs = writePython3 "test-writers-python3-no-libs" {} '' print("success") ''; }; path = { - bash = writeBash "test_bash" (writeText "test" '' + bash = writeBash "test-writers-bash-path" (writeText "test" '' if [[ "test" == "test" ]]; then echo "success"; fi ''); - haskell = writeHaskell "test_haskell" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" '' + haskell = writeHaskell "test-writers-haskell-path" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" '' import Data.Default int :: Int @@ -180,8 +184,8 @@ let ''); }; - writeTest = expectedValue: test: - writeDash "test-writers" '' + writeTest = expectedValue: name: test: + writeDash "run-${name}" '' if test "$(${test})" != "${expectedValue}"; then echo 'test ${test} failed' exit 1 @@ -189,12 +193,12 @@ let ''; in runCommand "test-writers" { - passthru = { inherit writeTest bin simple; }; - meta.platforms = stdenv.lib.platforms.all; + passthru = { inherit writeTest bin simple path; }; + meta.platforms = lib.platforms.all; } '' - ${lib.concatMapStringsSep "\n" (test: writeTest "success" "${test}/bin/test_writers") (lib.attrValues bin)} - ${lib.concatMapStringsSep "\n" (test: writeTest "success" test) (lib.attrValues simple)} - ${lib.concatMapStringsSep "\n" (test: writeTest "success" test) (lib.attrValues path)} + ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name "${test}/bin/${test.name}") (lib.attrValues bin)} + ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues simple)} + ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues path)} echo 'nix-writers successfully tested' >&2 touch $out |