diff options
Diffstat (limited to 'pkgs/build-support')
85 files changed, 1150 insertions, 742 deletions
diff --git a/pkgs/build-support/agda/default.nix b/pkgs/build-support/agda/default.nix index 63adf2a2765..893383a759a 100644 --- a/pkgs/build-support/agda/default.nix +++ b/pkgs/build-support/agda/default.nix @@ -47,6 +47,7 @@ let "lagda.org" "lagda.rst" "lagda.tex" + "lagda.typ" ]; defaults = diff --git a/pkgs/build-support/agda/lib.nix b/pkgs/build-support/agda/lib.nix index 80a0974192b..e6e391b2ced 100644 --- a/pkgs/build-support/agda/lib.nix +++ b/pkgs/build-support/agda/lib.nix @@ -6,7 +6,7 @@ * interfaceFile "Everything.agda" == "Everything.agdai" * interfaceFile "src/Everything.lagda.tex" == "src/Everything.agdai" */ - interfaceFile = agdaFile: lib.head (builtins.match ''(.*\.)l?agda(\.(md|org|rst|tex))?'' agdaFile) + "agdai"; + interfaceFile = agdaFile: lib.head (builtins.match ''(.*\.)l?agda(\.(md|org|rst|tex|typ))?'' agdaFile) + "agdai"; /* Takes an arbitrary derivation and says whether it is an agda library package * that is not marked as broken. diff --git a/pkgs/build-support/appimage/default.nix b/pkgs/build-support/appimage/default.nix index 0637964ca62..7b9bb239402 100644 --- a/pkgs/build-support/appimage/default.nix +++ b/pkgs/build-support/appimage/default.nix @@ -26,10 +26,11 @@ rec { ]; }; - extract = args@{ name ? "${args.pname}-${args.version}", src, ... }: pkgs.runCommand "${name}-extracted" { + extract = args@{ name ? "${args.pname}-${args.version}", postExtract ? "", src, ... }: pkgs.runCommand "${name}-extracted" { buildInputs = [ appimage-exec ]; } '' appimage-exec.sh -x $out ${src} + ${postExtract} ''; # for compatibility, deprecated @@ -206,6 +207,7 @@ rec { libtool.lib # for Synfigstudio xorg.libxshmfence # for apple-music-electron at-spi2-core + pciutils # for FreeCAD ]; }; } diff --git a/pkgs/build-support/binary-cache/default.nix b/pkgs/build-support/binary-cache/default.nix index 27f9ad96289..8c610c51227 100644 --- a/pkgs/build-support/binary-cache/default.nix +++ b/pkgs/build-support/binary-cache/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildPackages }: +{ lib, stdenv, coreutils, jq, python3, nix, xz }: # This function is for creating a flat-file binary cache, i.e. the kind created by # nix copy --to file:///some/path and usable as a substituter (with the file:// prefix). @@ -19,15 +19,10 @@ stdenv.mkDerivation { preferLocalBuild = true; - PATH = lib.makeBinPath (with buildPackages; [ coreutils jq python3 nix xz ]); + nativeBuildInputs = [ coreutils jq python3 nix xz ]; - builder = builtins.toFile "builder" '' - . .attrs.sh - - export out=''${outputs[out]} - - mkdir $out - mkdir $out/nar + buildCommand = '' + mkdir -p $out/nar python ${./make-binary-cache.py} diff --git a/pkgs/build-support/binary-cache/make-binary-cache.py b/pkgs/build-support/binary-cache/make-binary-cache.py index 16dd8a7e96b..589d005562b 100644 --- a/pkgs/build-support/binary-cache/make-binary-cache.py +++ b/pkgs/build-support/binary-cache/make-binary-cache.py @@ -3,7 +3,7 @@ import json import os import subprocess -with open(".attrs.json", "r") as f: +with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f: closures = json.load(f)["closure"] os.chdir(os.environ["out"]) diff --git a/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix b/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix index a6802f4ab54..b5e03164ac2 100644 --- a/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix +++ b/pkgs/build-support/build-fhsenv-bubblewrap/buildFHSEnv.nix @@ -83,7 +83,7 @@ let etcProfile = writeText "profile" '' export PS1='${name}-chrootenv:\u@\h:\w\$ ' export LOCALE_ARCHIVE='/usr/lib/locale/locale-archive' - export LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib:/usr/lib:/usr/lib32''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH" export PATH="/run/wrappers/bin:/usr/bin:/usr/sbin:$PATH" export TZDIR='/etc/zoneinfo' diff --git a/pkgs/build-support/build-fhsenv-bubblewrap/default.nix b/pkgs/build-support/build-fhsenv-bubblewrap/default.nix index 5e8894d7cb3..3500e5e9216 100644 --- a/pkgs/build-support/build-fhsenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhsenv-bubblewrap/default.nix @@ -31,7 +31,7 @@ assert (pname != null || version != null) -> (name == null && pname != null); # with builtins; let - pname = if args.name != null then args.name else args.pname; + pname = if args ? name && args.name != null then args.name else args.pname; versionStr = lib.optionalString (version != null) ("-" + version); name = pname + versionStr; @@ -223,6 +223,7 @@ let bin = writeShellScript "${name}-bwrap" (bwrapCmd { initArgs = ''"$@"''; }); in runCommandLocal name { + inherit pname version; inherit meta; passthru = passthru // { diff --git a/pkgs/build-support/build-graalvm-native-image/default.nix b/pkgs/build-support/build-graalvm-native-image/default.nix index c3c4a7b1e57..6a5e82427f7 100644 --- a/pkgs/build-support/build-graalvm-native-image/default.nix +++ b/pkgs/build-support/build-graalvm-native-image/default.nix @@ -3,7 +3,6 @@ , glibcLocales # The GraalVM derivation to use , graalvmDrv -, name ? "${args.pname}-${args.version}" , executable ? args.pname # JAR used as input for GraalVM derivation, defaults to src , jar ? args.src @@ -13,14 +12,15 @@ , nativeImageBuildArgs ? [ (lib.optionalString stdenv.isDarwin "-H:-CheckToolchain") "-H:Name=${executable}" + "-march=compatibility" "--verbose" + "-J-Dsun.stdout.encoding=UTF-8" + "-J-Dsun.stderr.encoding=UTF-8" ] # Extra arguments to be passed to the native-image , extraNativeImageBuildArgs ? [ ] # XMX size of GraalVM during build , graalvmXmx ? "-J-Xmx6g" - # Locale to be used by GraalVM compiler -, LC_ALL ? "en_US.UTF-8" , meta ? { } , ... } @ args: @@ -40,7 +40,7 @@ let ]; in stdenv.mkDerivation ({ - inherit dontUnpack LC_ALL jar; + inherit dontUnpack jar; nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ graalvmDrv glibcLocales ]; @@ -71,7 +71,5 @@ stdenv.mkDerivation ({ platforms = graalvmDrv.meta.platforms; # default to executable name mainProgram = executable; - # need to have native-image-installable-svm available - broken = !(builtins.any (p: (p.product or "") == "native-image-installable-svm") graalvmDrv.products); } // meta; } // extraArgs) diff --git a/pkgs/build-support/cc-wrapper/add-hardening.sh b/pkgs/build-support/cc-wrapper/add-hardening.sh index 8d02b4e5124..8cd63e46095 100644 --- a/pkgs/build-support/cc-wrapper/add-hardening.sh +++ b/pkgs/build-support/cc-wrapper/add-hardening.sh @@ -10,6 +10,13 @@ for flag in ${NIX_HARDENING_ENABLE_@suffixSalt@-}; do hardeningEnableMap["$flag"]=1 done +# fortify3 implies fortify enablement - make explicit before +# we filter unsupported flags because unsupporting fortify3 +# doesn't mean we should unsupport fortify too +if [[ -n "${hardeningEnableMap[fortify3]-}" ]]; then + hardeningEnableMap["fortify"]=1 +fi + # Remove unsupported flags. for flag in @hardening_unsupported_flags@; do unset -v "hardeningEnableMap[$flag]" @@ -19,7 +26,7 @@ for flag in @hardening_unsupported_flags@; do fi done -# make fortify and fortify3 mutually exclusive +# now make fortify and fortify3 mutually exclusive if [[ -n "${hardeningEnableMap[fortify3]-}" ]]; then unset -v "hardeningEnableMap['fortify']" fi diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh index 9dcd29c6443..b8d170df01b 100644 --- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh +++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh @@ -31,7 +31,6 @@ cxxLibrary=1 cInclude=1 expandResponseParams "$@" -linkType=$(checkLinkType "${params[@]}") declare -ag positionalArgs=() declare -i n=0 @@ -175,6 +174,7 @@ extraAfter=(${hardeningCFlagsAfter[@]+"${hardeningCFlagsAfter[@]}"} $NIX_CFLAGS_ extraBefore=(${hardeningCFlagsBefore[@]+"${hardeningCFlagsBefore[@]}"} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@) if [ "$dontLink" != 1 ]; then + linkType=$(checkLinkType $NIX_LDFLAGS_BEFORE_@suffixSalt@ "${params[@]}" ${NIX_CFLAGS_LINK_@suffixSalt@:-} $NIX_LDFLAGS_@suffixSalt@) # Add the flags that should only be passed to the compiler when # linking. @@ -246,7 +246,7 @@ if [[ -e @out@/nix-support/cc-wrapper-hook ]]; then fi if (( "${NIX_CC_USE_RESPONSE_FILE:-@use_response_file_by_default@}" >= 1 )); then - responseFile=$(mktemp --tmpdir cc-params.XXXXXX) + responseFile=$(mktemp "${TMPDIR:-/tmp}/cc-params.XXXXXX") trap 'rm -f -- "$responseFile"' EXIT printf "%q\n" \ ${extraBefore+"${extraBefore[@]}"} \ diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index e1da3ceb5bc..8ac11436c5f 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -110,7 +110,20 @@ let gccForLibs_solib = getLib gccForLibs + optionalString (targetPlatform != hostPlatform) "/${targetPlatform.config}"; - # older compilers (for example bootstrap's GCC 5) fail with -march=too-modern-cpu + # The following two functions, `isGccArchSupported` and + # `isGccTuneSupported`, only handle those situations where a flag + # (`-march` or `-mtune`) is accepted by one compiler but rejected + # by another, and both compilers are relevant to nixpkgs. We are + # not trying to maintain a complete list of all flags accepted by + # all versions of all compilers ever in nixpkgs. + # + # The two main cases of interest are: + # + # - One compiler is gcc and the other is clang + # - One compiler is pkgs.gcc and the other is bootstrap-files.gcc + # -- older compilers (for example bootstrap's GCC 5) fail with + # -march=too-modern-cpu + isGccArchSupported = arch: if targetPlatform.isPower then false else # powerpc does not allow -march= if isGNU then @@ -159,6 +172,51 @@ let else false; + isGccTuneSupported = tune: + # for x86 -mtune= takes the same values as -march, plus two more: + if targetPlatform.isx86 then + { + generic = true; + intel = true; + }.${tune} or (isGccArchSupported tune) + # on arm64, the -mtune= values are specific processors + else if targetPlatform.isAarch64 then + (if isGNU then + { + cortex-a53 = versionAtLeast ccVersion "4.8"; # gcc 8c075f + cortex-a72 = versionAtLeast ccVersion "5.1"; # gcc d8f70d + "cortex-a72.cortex-a53" = versionAtLeast ccVersion "5.1"; # gcc d8f70d + }.${tune} or false + else if isClang then + { + cortex-a53 = versionAtLeast ccVersion "3.9"; # llvm dfc5d1 + }.${tune} or false + else false) + else if targetPlatform.isPower then + # powerpc does not support -march + true + else if targetPlatform.isMips then + # for mips -mtune= takes the same values as -march + isGccArchSupported tune + else + false; + + # Clang does not support as many `-mtune=` values as gcc does; + # this function will return the best possible approximation of the + # provided `-mtune=` value, or `null` if none exists. + # + # Note: this function can make use of ccVersion; for example, `if + # versionOlder ccVersion "12" then ...` + findBestTuneApproximation = tune: + let guess = if isClang + then { + # clang does not tune for big.LITTLE chips + "cortex-a72.cortex-a53" = "cortex-a72"; + }.${tune} or tune + else tune; + in if isGccTuneSupported guess + then guess + else null; darwinPlatformForCC = optionalString stdenv.targetPlatform.isDarwin ( if (targetPlatform.darwinPlatform == "macos" && isGNU) then "macosx" @@ -469,6 +527,7 @@ stdenv.mkDerivation { '' + optionalString (libcxx.isLLVM or false) '' echo "-isystem ${lib.getDev libcxx}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags + echo "-isystem ${lib.getDev libcxx.cxxabi}/include/c++/v1" >> $out/nix-support/libcxx-cxxflags echo "-stdlib=libc++" >> $out/nix-support/libcxx-ldflags echo "-l${libcxx.cxxabi.libName}" >> $out/nix-support/libcxx-ldflags '' @@ -558,10 +617,12 @@ stdenv.mkDerivation { + optionalString (targetPlatform ? gcc.thumb) '' echo "-m${if targetPlatform.gcc.thumb then "thumb" else "arm"}" >> $out/nix-support/cc-cflags-before '' - + optionalString (targetPlatform ? gcc.tune && - isGccArchSupported targetPlatform.gcc.tune) '' - echo "-mtune=${targetPlatform.gcc.tune}" >> $out/nix-support/cc-cflags-before - '' + + (let tune = if targetPlatform ? gcc.tune + then findBestTuneApproximation targetPlatform.gcc.tune + else null; + in optionalString (tune != null) '' + echo "-mtune=${tune}" >> $out/nix-support/cc-cflags-before + '') # TODO: categorize these and figure out a better place for them + optionalString targetPlatform.isWindows '' diff --git a/pkgs/build-support/closure-info.nix b/pkgs/build-support/closure-info.nix index 6b3ff6fd62b..f2aa4964d9a 100644 --- a/pkgs/build-support/closure-info.nix +++ b/pkgs/build-support/closure-info.nix @@ -4,7 +4,7 @@ # "nix-store --load-db" and "nix-store --register-validity # --hash-given". -{ stdenv, buildPackages }: +{ stdenv, coreutils, jq }: { rootPaths }: @@ -19,18 +19,24 @@ stdenv.mkDerivation { preferLocalBuild = true; - PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin"; + nativeBuildInputs = [ coreutils jq ]; - builder = builtins.toFile "builder" - '' - . .attrs.sh + empty = rootPaths == []; + buildCommand = + '' out=''${outputs[out]} mkdir $out - jq -r ".closure | map(.narSize) | add" < .attrs.json > $out/total-nar-size - jq -r '.closure | map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < .attrs.json | head -n -1 > $out/registration - jq -r .closure[].path < .attrs.json > $out/store-paths + if [[ -n "$empty" ]]; then + echo 0 > $out/total-nar-size + touch $out/registration $out/store-paths + else + jq -r ".closure | map(.narSize) | add" < "$NIX_ATTRS_JSON_FILE" > $out/total-nar-size + jq -r '.closure | map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < "$NIX_ATTRS_JSON_FILE" | head -n -1 > $out/registration + jq -r '.closure[].path' < "$NIX_ATTRS_JSON_FILE" > $out/store-paths + fi + ''; } diff --git a/pkgs/build-support/dart/build-dart-application/default.nix b/pkgs/build-support/dart/build-dart-application/default.nix index be1fd727767..1a98f98fb8e 100644 --- a/pkgs/build-support/dart/build-dart-application/default.nix +++ b/pkgs/build-support/dart/build-dart-application/default.nix @@ -1,6 +1,7 @@ -{ lib, stdenv, fetchDartDeps, runCommand, writeText, dartHooks, makeWrapper, dart, cacert, nodejs, darwin }: +{ lib, stdenv, callPackage, fetchDartDeps, writeText, symlinkJoin, dartHooks, makeWrapper, dart, cacert, nodejs, darwin, jq }: -{ pubGetScript ? "dart pub get" +{ sdkSetupScript ? "" +, pubGetScript ? "dart pub get" # Output type to produce. Can be any kind supported by dart # https://dart.dev/tools/dart-compile#types-of-output @@ -18,12 +19,16 @@ , dartEntryPoints ? null # Used when wrapping aot, jit, kernel, and js builds. # Set to null to disable wrapping. -, dartRuntimeCommand ? - if dartOutputType == "aot-snapshot" then "${dart}/bin/dartaotruntime" - else if (dartOutputType == "jit-snapshot" || dartOutputType == "kernel") then "${dart}/bin/dart" - else if dartOutputType == "js" then "${nodejs}/bin/node" - else null +, dartRuntimeCommand ? if dartOutputType == "aot-snapshot" then "${dart}/bin/dartaotruntime" + else if (dartOutputType == "jit-snapshot" || dartOutputType == "kernel") then "${dart}/bin/dart" + else if dartOutputType == "js" then "${nodejs}/bin/node" + else null +, runtimeDependencies ? [ ] +, extraWrapProgramArgs ? "" +, customPackageOverrides ? { } +, autoDepsList ? false +, depsListFile ? null , pubspecLockFile ? null , vendorHash ? "" , ... @@ -31,44 +36,92 @@ let dartDeps = (fetchDartDeps.override { - dart = runCommand "dart-fod" { nativeBuildInputs = [ makeWrapper ]; } '' - mkdir -p "$out/bin" - makeWrapper "${dart}/bin/dart" "$out/bin/dart" \ - --add-flags "--root-certs-file=${cacert}/etc/ssl/certs/ca-bundle.crt" - ''; + dart = symlinkJoin { + name = "dart-fod"; + paths = [ dart ]; + nativeBuildInputs = [ makeWrapper ]; + postBuild = '' + wrapProgram "$out/bin/dart" \ + --add-flags "--root-certs-file=${cacert}/etc/ssl/certs/ca-bundle.crt" + ''; + }; }) { buildDrvArgs = args; - inherit pubGetScript vendorHash pubspecLockFile; + inherit sdkSetupScript pubGetScript vendorHash pubspecLockFile; }; - inherit (dartHooks.override { inherit dart; }) dartConfigHook dartBuildHook dartInstallHook; -in -assert !(builtins.isString dartOutputType && dartOutputType != "") -> - throw "dartOutputType must be a non-empty string"; -stdenv.mkDerivation (args // { - inherit pubGetScript dartCompileCommand dartOutputType dartRuntimeCommand - dartCompileFlags dartJitFlags; + inherit (dartHooks.override { inherit dart; }) dartConfigHook dartBuildHook dartInstallHook dartFixupHook; + + baseDerivation = stdenv.mkDerivation (finalAttrs: args // { + inherit sdkSetupScript pubGetScript dartCompileCommand dartOutputType + dartRuntimeCommand dartCompileFlags dartJitFlags runtimeDependencies; dartEntryPoints = if (dartEntryPoints != null) then writeText "entrypoints.json" (builtins.toJSON dartEntryPoints) else null; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ - dart - dartDeps - dartConfigHook - dartBuildHook - dartInstallHook - makeWrapper - ] ++ lib.optionals stdenv.isDarwin [ - darwin.sigtool - ]; + runtimeDependencyLibraryPath = lib.makeLibraryPath finalAttrs.runtimeDependencies; + + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ + dart + dartDeps + dartConfigHook + dartBuildHook + dartInstallHook + dartFixupHook + makeWrapper + jq + ] ++ lib.optionals stdenv.isDarwin [ + darwin.sigtool + ]; - # When stripping, it seems some ELF information is lost and the dart VM cli - # runs instead of the expected program. Don't strip if it's an exe output. - dontStrip = args.dontStrip or (dartOutputType == "exe"); + preUnpack = '' + ${lib.optionalString (!autoDepsList) '' + if ! { [ '${lib.boolToString (depsListFile != null)}' = 'true' ] ${lib.optionalString (depsListFile != null) "&& cmp -s <(jq -Sc . '${depsListFile}') <(jq -Sc . '${finalAttrs.passthru.dartDeps.depsListFile}')"}; }; then + echo 1>&2 -e '\nThe dependency list file was either not given or differs from the expected result.' \ + '\nPlease choose one of the following solutions:' \ + '\n - Duplicate the following file and pass it to the depsListFile argument.' \ + '\n ${finalAttrs.passthru.dartDeps.depsListFile}' \ + '\n - Set autoDepsList to true (not supported by Hydra or permitted in Nixpkgs)'. + exit 1 + fi + ''} + ${args.preUnpack or ""} + ''; + + # When stripping, it seems some ELF information is lost and the dart VM cli + # runs instead of the expected program. Don't strip if it's an exe output. + dontStrip = args.dontStrip or (dartOutputType == "exe"); + + passthru = { inherit dartDeps; } // (args.passthru or { }); - passthru = { inherit dartDeps; } // (args.passthru or { }); + meta = (args.meta or { }) // { platforms = args.meta.platforms or dart.meta.platforms; }; + }); - meta = (args.meta or { }) // { platforms = args.meta.platforms or dart.meta.platforms; }; -}) + packageOverrideRepository = (callPackage ../../../development/compilers/dart/package-overrides { }) // customPackageOverrides; + productPackages = builtins.filter (package: package.kind != "dev") + (if autoDepsList + then lib.importJSON dartDeps.depsListFile + else + if depsListFile == null + then [ ] + else lib.importJSON depsListFile); +in +assert !(builtins.isString dartOutputType && dartOutputType != "") -> +throw "dartOutputType must be a non-empty string"; +builtins.foldl' + (prev: package: + if packageOverrideRepository ? ${package.name} + then + prev.overrideAttrs + (packageOverrideRepository.${package.name} { + inherit (package) + name + version + kind + source + dependencies; + }) + else prev) + baseDerivation + productPackages diff --git a/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh b/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh index 3e901995237..f22d7d2ce64 100644 --- a/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh +++ b/pkgs/build-support/dart/build-dart-application/hooks/dart-config-hook.sh @@ -3,6 +3,9 @@ dartConfigHook() { echo "Executing dartConfigHook" + echo "Setting up SDK" + eval "$sdkSetupScript" + echo "Installing dependencies" eval doPubGet "$pubGetScript" --offline diff --git a/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh b/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh new file mode 100644 index 00000000000..c5a9bedd066 --- /dev/null +++ b/pkgs/build-support/dart/build-dart-application/hooks/dart-fixup-hook.sh @@ -0,0 +1,32 @@ +# shellcheck shell=bash + +dartFixupHook() { + echo "Executing dartFixupHook" + + declare -a wrapProgramArgs + + # Add runtime library dependencies to the LD_LIBRARY_PATH. + # For some reason, the RUNPATH of the executable is not used to load dynamic libraries in dart:ffi with DynamicLibrary.open(). + # + # This could alternatively be fixed with patchelf --add-needed, but this would cause all the libraries to be opened immediately, + # which is not what application authors expect. + echo "$runtimeDependencyLibraryPath" + if [[ ! -z "$runtimeDependencyLibraryPath" ]]; then + wrapProgramArgs+=(--suffix LD_LIBRARY_PATH : \"$runtimeDependencyLibraryPath\") + fi + + if [[ ! -z "$extraWrapProgramArgs" ]]; then + wrapProgramArgs+=("$extraWrapProgramArgs") + fi + + if [ ${#wrapProgramArgs[@]} -ne 0 ]; then + for f in "$out"/bin/*; do + echo "Wrapping $f..." + eval "wrapProgram \"$f\" ${wrapProgramArgs[@]}" + done + fi + + echo "Finished dartFixupHook" +} + +postFixupHooks+=(dartFixupHook) diff --git a/pkgs/build-support/dart/build-dart-application/hooks/default.nix b/pkgs/build-support/dart/build-dart-application/hooks/default.nix index 463061c54a8..134989426d9 100644 --- a/pkgs/build-support/dart/build-dart-application/hooks/default.nix +++ b/pkgs/build-support/dart/build-dart-application/hooks/default.nix @@ -12,4 +12,7 @@ dartInstallHook = makeSetupHook { name = "dart-install-hook"; } ./dart-install-hook.sh; + dartFixupHook = makeSetupHook { + name = "dart-fixup-hook"; + } ./dart-fixup-hook.sh; } diff --git a/pkgs/build-support/dart/fetch-dart-deps/default.nix b/pkgs/build-support/dart/fetch-dart-deps/default.nix index e523b60797e..51052cae18f 100644 --- a/pkgs/build-support/dart/fetch-dart-deps/default.nix +++ b/pkgs/build-support/dart/fetch-dart-deps/default.nix @@ -169,6 +169,8 @@ let dart pub deps --json | jq .packages > $out runHook postBuild ''; + + dontInstall = true; } // (removeAttrs buildDrvInheritArgs [ "name" "pname" ])); # As of Dart 3.0.0, Pub checks the revision of cached Git-sourced packages. diff --git a/pkgs/build-support/deterministic-uname/deterministic-uname.sh b/pkgs/build-support/deterministic-uname/deterministic-uname.sh index 5272bb5b3fe..31772aeee3c 100644 --- a/pkgs/build-support/deterministic-uname/deterministic-uname.sh +++ b/pkgs/build-support/deterministic-uname/deterministic-uname.sh @@ -38,6 +38,10 @@ processor=0 hardware_platform=0 operating_system=0 +# With no OPTION, same as -s. +if [[ $# -eq 0 ]]; then + kernel_name=1 +fi @getopt@/bin/getopt --test > /dev/null && rc=$? || rc=$? if [[ $rc -ne 4 ]]; then @@ -54,11 +58,6 @@ else eval set -- "$PARSED" fi -# With no OPTION, same as -s. -if [[ $# -eq 0 ]]; then - kernel_name=1 -fi - # Process each argument, and set the appropriate flag if we recognize it. while [[ $# -ge 1 ]]; do case "$1" in @@ -132,44 +131,44 @@ fi # Darwin *nodename* 22.1.0 Darwin Kernel Version 22.1.0: Sun Oct 9 20:14:30 PDT 2022; root:xnu-8792.41.9~2/RELEASE_ARM64_T8103 arm64 arm Darwin # NixOS: # Linux *nodename* 6.0.13 #1-NixOS SMP PREEMPT_DYNAMIC Wed Dec 14 10:41:06 UTC 2022 x86_64 GNU/Linux +output=() if [[ "$all" = "1" ]]; then - echo -n "$KERNEL_NAME_VAL $NODENAME_VAL $KERNEL_RELEASE_VAL $KERNEL_VERSION_VAL $MACHINE_VAL " + output+=("$KERNEL_NAME_VAL" "$NODENAME_VAL" "$KERNEL_RELEASE_VAL" "$KERNEL_VERSION_VAL" "$MACHINE_VAL") # in help: except omit -p and -i if unknown. - #echo -n "$PROCESSOR_VAL $HARDWARE_PLATFORM_VAL\n" - echo -n "$OPERATING_SYSTEM_VAL" + # output+=($PROCESSOR_VAL $HARDWARE_PLATFORM_VAL) + output+=("$OPERATING_SYSTEM_VAL") fi if [[ "$kernel_name" = "1" ]]; then - echo -n "$KERNEL_NAME_VAL" + output+=("$KERNEL_NAME_VAL") fi if [[ "$nodename" = "1" ]]; then - echo -n "$NODENAME_VAL" + output+=("$NODENAME_VAL") fi if [[ "$kernel_release" = "1" ]]; then - echo -n "$KERNEL_RELEASE_VAL" + output+=("$KERNEL_RELEASE_VAL") fi if [[ "$kernel_version" = "1" ]]; then - echo -n "$KERNEL_VERSION_VAL" + output+=("$KERNEL_VERSION_VAL") fi if [[ "$machine" = "1" ]]; then - echo -n "$MACHINE_VAL" + output+=("$MACHINE_VAL") fi if [[ "$processor" = "1" ]]; then - echo -n "$PROCESSOR_VAL" + output+=("$PROCESSOR_VAL") fi if [[ "$hardware_platform" = "1" ]]; then - echo -n "$HARDWARE_PLATFORM_VAL" + output+=("$HARDWARE_PLATFORM_VAL") fi if [[ "$operating_system" = "1" ]]; then - echo -n "$OPERATING_SYSTEM_VAL" + output+=("$OPERATING_SYSTEM_VAL") fi -# for newline. -echo +echo "${output[@]}" diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 1ac0a69f745..70fd3635b74 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -446,7 +446,7 @@ rec { ''; postMount = '' - mkdir -p mnt/{dev,proc,sys} mnt${storeDir} + mkdir -p mnt/{dev,proc,sys,tmp} mnt${storeDir} # Mount /dev, /sys and the nix store as shared folders. mount --rbind /dev mnt/dev diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index 686d89f8c11..f8ed3a38890 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -158,6 +158,22 @@ stdenvNoCC.mkDerivation (args // { dotnet-sdk ]; + # Parse the version attr into a format acceptable for the Version msbuild property + # The actual version attr is saved in InformationalVersion, which accepts an arbitrary string + versionForDotnet = if !(lib.hasAttr "version" args) || args.version == null + then null else let + components = lib.pipe args.version [ + lib.splitVersion + (lib.filter (x: (lib.strings.match "[0-9]+" x) != null)) + (lib.filter (x: (lib.toInt x) < 65535)) # one version component in dotnet has to fit in 16 bits + ]; + in if (lib.length components) == 0 + then null + else lib.concatStringsSep "." ((lib.take 4 components) + ++ (if (lib.length components) < 4 + then lib.replicate (4 - (lib.length components)) "0" + else [ ])); + makeWrapperArgs = args.makeWrapperArgs or [ ] ++ [ "--prefix LD_LIBRARY_PATH : ${dotnet-sdk.icu}/lib" ]; @@ -172,7 +188,7 @@ stdenvNoCC.mkDerivation (args // { passthru = { inherit nuget-source; - } // lib.optionalAttrs (nugetDepsFile != null) { + } // lib.optionalAttrs (!lib.isDerivation nugetDeps) { fetch-deps = let flags = dotnetFlags ++ dotnetRestoreFlags; diff --git a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh index e9567b64cf2..0acfeced9b1 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh +++ b/pkgs/build-support/dotnet/build-dotnet-module/hooks/dotnet-build-hook.sh @@ -24,8 +24,13 @@ dotnetBuildHook() { dotnetBuildFlags+=("-p:UseAppHost=true") fi + local versionFlags=() if [ "${version-}" ]; then - local -r versionFlag="-p:Version=${version-}" + versionFlags+=("-p:InformationalVersion=${version-}") + fi + + if [ "${versionForDotnet-}" ]; then + versionFlags+=("-p:Version=${versionForDotnet-}") fi dotnetBuild() { @@ -43,7 +48,7 @@ dotnetBuildHook() { -p:Deterministic=true \ --configuration "@buildType@" \ --no-restore \ - ${versionFlag-} \ + ${versionFlags[@]} \ ${runtimeIdFlags[@]} \ ${dotnetBuildFlags[@]} \ ${dotnetFlags[@]} diff --git a/pkgs/build-support/expand-response-params/default.nix b/pkgs/build-support/expand-response-params/default.nix index 9371b770236..7ce15e98c8d 100644 --- a/pkgs/build-support/expand-response-params/default.nix +++ b/pkgs/build-support/expand-response-params/default.nix @@ -1,5 +1,12 @@ { stdenv }: +# A "response file" is a sequence of arguments that is passed via a +# file, rather than via argv[]. + +# For more information see: +# https://gcc.gnu.org/wiki/Response_Files +# https://www.intel.com/content/www/us/en/docs/dpcpp-cpp-compiler/developer-guide-reference/2023-0/use-response-files.html + stdenv.mkDerivation { name = "expand-response-params"; src = ./expand-response-params.c; diff --git a/pkgs/build-support/fetchbzr/builder.sh b/pkgs/build-support/fetchbzr/builder.sh index 991864719a0..4515e0e82f4 100644 --- a/pkgs/build-support/fetchbzr/builder.sh +++ b/pkgs/build-support/fetchbzr/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source "$stdenv/setup" echo "exporting \`$url' (revision $rev) into \`$out'" diff --git a/pkgs/build-support/fetchcvs/builder.sh b/pkgs/build-support/fetchcvs/builder.sh index 4b49e9676ec..7e4dde4a64c 100644 --- a/pkgs/build-support/fetchcvs/builder.sh +++ b/pkgs/build-support/fetchcvs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup (echo "#!$SHELL"; \ diff --git a/pkgs/build-support/fetchdarcs/builder.sh b/pkgs/build-support/fetchdarcs/builder.sh index 75b9066dba6..39838db255a 100644 --- a/pkgs/build-support/fetchdarcs/builder.sh +++ b/pkgs/build-support/fetchdarcs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup tagtext="" diff --git a/pkgs/build-support/fetchdocker/credentials.nix b/pkgs/build-support/fetchdocker/credentials.nix index da198483268..f8a229ccb6b 100644 --- a/pkgs/build-support/fetchdocker/credentials.nix +++ b/pkgs/build-support/fetchdocker/credentials.nix @@ -1,3 +1,4 @@ +{ lib }: # We provide three paths to get the credentials into the builder's # environment: # diff --git a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh index 4eb70f672d4..489914a2a8b 100644 --- a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh +++ b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source "${stdenv}/setup" echo "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}" mkdir -p "${out}" diff --git a/pkgs/build-support/fetchdocker/generic-fetcher.nix b/pkgs/build-support/fetchdocker/generic-fetcher.nix index 6a7b977db29..95b193490a8 100644 --- a/pkgs/build-support/fetchdocker/generic-fetcher.nix +++ b/pkgs/build-support/fetchdocker/generic-fetcher.nix @@ -1,7 +1,7 @@ { stdenv, lib, haskellPackages, writeText, gawk }: let awk = "${gawk}/bin/awk"; - dockerCredentialsFile = import ./credentials.nix; + dockerCredentialsFile = import ./credentials.nix { inherit lib; }; in { fetcher , name diff --git a/pkgs/build-support/fetchfossil/builder.sh b/pkgs/build-support/fetchfossil/builder.sh index 36b758ab574..fe828d0ada9 100644 --- a/pkgs/build-support/fetchfossil/builder.sh +++ b/pkgs/build-support/fetchfossil/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "Cloning Fossil $url [$rev] into $out" diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 77f6381b09a..95176beea8c 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -3,7 +3,7 @@ # - revision specified and remote has a HEAD # - revision specified and remote without HEAD # -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (rev $rev) into $out" diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 1d06ce44a91..66bb3b7c09f 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -90,6 +90,7 @@ stdenvNoCC.mkDerivation { ${netrcPhase} # required that git uses the netrc file mv {,.}netrc + export NETRC=$PWD/.netrc export HOME=$PWD ''; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index 2a53fd94e7f..a3ea0297673 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -207,7 +207,10 @@ checkout_ref(){ # Update submodules init_submodules(){ - clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} + # shallow with leaveDotGit will change hashes + [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \ + clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \ + clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress } clone(){ @@ -257,9 +260,15 @@ make_deterministic_repo(){ cd "$repo" # Remove files that contain timestamps or otherwise have non-deterministic # properties. - rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \ - .git/refs/remotes/origin/HEAD .git/config - + if [ -f .git ]; then + local dotgit_content=$(<.git) + local dotgit_dir="${dotgit_content#gitdir: }" + else + local dotgit_dir=".git" + fi + pushd "$dotgit_dir" >/dev/null + rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config + popd >/dev/null # Remove all remote branches. git branch -r | while read -r branch; do clean_git branch -rD "$branch" @@ -277,7 +286,7 @@ make_deterministic_repo(){ # Do a full repack. Must run single-threaded, or else we lose determinism. clean_git config pack.threads 1 clean_git repack -A -d -f - rm -f .git/config + rm -f "$dotgit_dir/config" # Garbage collect unreferenced objects. # Note: --keep-largest-pack prevents non-deterministic ordering of packs @@ -293,9 +302,6 @@ clone_user_rev() { local rev="${3:-HEAD}" if [ -n "$fetchLFS" ]; then - tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")" - exit_handlers+=(remove_tmpHomePath) - HOME="$tmpHomePath" clean_git lfs install fi @@ -326,7 +332,7 @@ clone_user_rev() { find "$dir" -name .git -print0 | xargs -0 rm -rf else find "$dir" -name .git | while read -r gitdir; do - make_deterministic_repo "$(readlink -f "$gitdir/..")" + make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")" done fi } @@ -417,6 +423,13 @@ if test -z "$branchName"; then branchName=fetchgit fi +tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")" +exit_handlers+=(remove_tmpHomePath) +ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc" +HOME="$tmpHomePath" +unset XDG_CONFIG_HOME +export GIT_CONFIG_NOSYSTEM=1 + if test -n "$builder"; then test -n "$out" -a -n "$url" -a -n "$rev" || usage mkdir -p "$out" diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index a18be65327b..23e5fa29901 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -1,15 +1,13 @@ -{ testers, fetchgit, ... }: - -{ +{ testers, fetchgit, ... }: { simple = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "simple-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sparseCheckout = [ @@ -20,7 +18,7 @@ }; sparseCheckoutNonConeMode = testers.invalidateFetcherByDrvHash fetchgit { - name = "nix-source"; + name = "sparse-checkout-non-cone-nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sparseCheckout = [ @@ -30,4 +28,48 @@ nonConeMode = true; sha256 = "sha256-FknO6C/PSnMPfhUqObD4vsW4PhkwdmPa9blNzcNvJQ4="; }; + + leave-git = testers.invalidateFetcherByDrvHash fetchgit { + name = "leave-git-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + sha256 = "sha256-zZxDxqaeWvuWuzwPizBLR7d59zP24+zqnWllNICenko="; + leaveDotGit = true; + }; + + submodule-simple = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-simple-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-rmP8PQT0wJBopdtr/hsB7Y/L1G+ZPdHC2r9LB05Qrj4="; + fetchSubmodules = true; + }; + + submodule-leave-git = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-leave-git-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-+uXIClcRJ4S1rdgx2Oyww+Jv4h1VXp8tfeh9lb07Fhk="; + leaveDotGit = true; + fetchSubmodules = true; + }; + + submodule-deep = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-deep-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-LL7uhXQk3N3DcvBBxwjmfVx55tTXCGQ19T91tknopzw="; + deepClone = true; + fetchSubmodules = true; + }; + + submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-leave-git-deep-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + sha256 = "sha256-LL7uhXQk3N3DcvBBxwjmfVx55tTXCGQ19T91tknopzw="; + deepClone = true; + leaveDotGit = true; + fetchSubmodules = true; + }; } diff --git a/pkgs/build-support/fetchhg/builder.sh b/pkgs/build-support/fetchhg/builder.sh index 1ce29475771..20dfde4b10d 100644 --- a/pkgs/build-support/fetchhg/builder.sh +++ b/pkgs/build-support/fetchhg/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "getting $url${rev:+ ($rev)} into $out" diff --git a/pkgs/build-support/fetchipfs/builder.sh b/pkgs/build-support/fetchipfs/builder.sh index ca77962b538..1ddd111b151 100644 --- a/pkgs/build-support/fetchipfs/builder.sh +++ b/pkgs/build-support/fetchipfs/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup # Curl flags to handle redirects, not use EPSV, handle cookies for diff --git a/pkgs/build-support/fetchmtn/builder.sh b/pkgs/build-support/fetchmtn/builder.sh index 1aabd7949ee..de929fad55a 100644 --- a/pkgs/build-support/fetchmtn/builder.sh +++ b/pkgs/build-support/fetchmtn/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup set -x diff --git a/pkgs/build-support/fetchsvn/builder.sh b/pkgs/build-support/fetchsvn/builder.sh index aa4d049aba4..e187747f14e 100644 --- a/pkgs/build-support/fetchsvn/builder.sh +++ b/pkgs/build-support/fetchsvn/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (r$rev) into $out" diff --git a/pkgs/build-support/fetchsvnssh/builder.sh b/pkgs/build-support/fetchsvnssh/builder.sh index 5782151524f..e553446346d 100644 --- a/pkgs/build-support/fetchsvnssh/builder.sh +++ b/pkgs/build-support/fetchsvnssh/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup echo "exporting $url (r$rev) into $out" diff --git a/pkgs/build-support/fetchtorrent/default.nix b/pkgs/build-support/fetchtorrent/default.nix new file mode 100644 index 00000000000..beb2415df70 --- /dev/null +++ b/pkgs/build-support/fetchtorrent/default.nix @@ -0,0 +1,60 @@ +{ lib, runCommand, transmission_noSystemd, rqbit, writeShellScript, formats, cacert, rsync }: +let + urlRegexp = ''.*xt=urn:bt[im]h:([^&]{64}|[^&]{40}).*''; +in +{ url +, name ? + if (builtins.match urlRegexp url) == null then + "bittorrent" + else + "bittorrent-" + builtins.head (builtins.match urlRegexp url) +, config ? if (backend == "transmission") then { } else throw "json config for configuring fetchFromBitorrent only works with the transmission backend" +, hash +, backend ? "transmission" +, recursiveHash ? true +, postFetch ? "" +, postUnpack ? "" +}: +let + afterSuccess = writeShellScript "fetch-bittorrent-done.sh" '' + ${postUnpack} + # Flatten the directory, so that only the torrent contents are in $out, not + # the folder name + shopt -s dotglob + mv -v $downloadedDirectory/*/* $out + rm -v -rf $downloadedDirectory + unset downloadedDirectory + ${postFetch} + kill $PPID + ''; + jsonConfig = (formats.json {}).generate "jsonConfig" config; +in +runCommand name { + nativeBuildInputs = [ cacert ] ++ (if (backend == "transmission" ) then [ transmission_noSystemd ] else if (backend == "rqbit") then [ rqbit ] else throw "rqbit or transmission are the only available backends for fetchtorrent"); + outputHashAlgo = if hash != "" then null else "sha256"; + outputHash = hash; + outputHashMode = if recursiveHash then "recursive" else "flat"; + + # url will be written to the derivation, meaning it can be parsed and utilized + # by external tools, such as tools that may want to seed fetchtorrent calls + # in nixpkgs + inherit url; +} +(if (backend == "transmission") then '' + export HOME=$TMP + export downloadedDirectory=$out/downloadedDirectory + mkdir -p $downloadedDirectory + mkdir -p $HOME/.config/transmission + cp ${jsonConfig} $HOME/.config/transmission/settings.json + function handleChild { + # This detects failures and logs the contents of the transmission fetch + find $out + exit 0 + } + trap handleChild CHLD + transmission-cli --port $(shuf -n 1 -i 49152-65535) --portmap --finish ${afterSuccess} --download-dir $downloadedDirectory --config-dir "$HOME"/.config/transmission "$url" +'' else +'' + export HOME=$TMP + rqbit --disable-dht-persistence --http-api-listen-addr "127.0.0.1:$(shuf -n 1 -i 49152-65535)" download -o $out --exit-on-finish "$url" +'') diff --git a/pkgs/build-support/fetchtorrent/tests.nix b/pkgs/build-support/fetchtorrent/tests.nix new file mode 100644 index 00000000000..b36c82c6326 --- /dev/null +++ b/pkgs/build-support/fetchtorrent/tests.nix @@ -0,0 +1,25 @@ +{ testers, fetchtorrent, ... }: + +{ + http-link = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + }; + magnet-link = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "transmission"; + }; + http-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "https://webtorrent.io/torrents/wired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + }; + magnet-link-rqbit = testers.invalidateFetcherByDrvHash fetchtorrent { + url = "magnet:?xt=urn:btih:a88fda5954e89178c372716a6a78b8180ed4dad3&dn=The+WIRED+CD+-+Rip.+Sample.+Mash.+Share&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fwired-cd.torrent"; + hash = "sha256-OCsC22WuanqoN6lPv5wDT5ZxPcEHDpZ1EgXGvz1SDYo="; + backend = "rqbit"; + }; +} + diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index e8eaba93485..a82728ef102 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi source $stdenv/setup source $mirrorsFile diff --git a/pkgs/build-support/flutter/default.nix b/pkgs/build-support/flutter/default.nix index a0ed1211ed8..3e136211655 100644 --- a/pkgs/build-support/flutter/default.nix +++ b/pkgs/build-support/flutter/default.nix @@ -1,34 +1,28 @@ { lib , callPackage -, stdenvNoCC , runCommand , makeWrapper , wrapGAppsHook -, llvmPackages_13 +, fetchDartDeps +, buildDartApplication , cacert , glib , flutter -, jq }: # absolutely no mac support for now { pubGetScript ? "flutter pub get" , flutterBuildFlags ? [ ] -, runtimeDependencies ? [ ] -, customPackageOverrides ? { } -, autoDepsList ? false -, depsListFile ? null -, vendorHash ? "" -, pubspecLockFile ? null -, nativeBuildInputs ? [ ] -, preUnpack ? "" -, postFixup ? "" , extraWrapProgramArgs ? "" , ... }@args: -let - flutterSetupScript = '' + +(buildDartApplication.override { + dart = flutter; + fetchDartDeps = fetchDartDeps.override { dart = flutter; }; +}) (args // { + sdkSetupScript = '' # Pub needs SSL certificates. Dart normally looks in a hardcoded path. # https://github.com/dart-lang/sdk/blob/3.1.0/runtime/bin/security_context_linux.cc#L48 # @@ -54,136 +48,56 @@ let flutter config --enable-linux-desktop >/dev/null ''; - deps = callPackage ../dart/fetch-dart-deps { dart = flutter; } { - sdkSetupScript = flutterSetupScript; - inherit pubGetScript vendorHash pubspecLockFile; - buildDrvArgs = args; - }; - - baseDerivation = llvmPackages_13.stdenv.mkDerivation (finalAttrs: args // { - inherit flutterBuildFlags runtimeDependencies; - - outputs = [ "out" "debug" ]; - - nativeBuildInputs = [ - makeWrapper - deps - flutter - jq - glib - wrapGAppsHook - ] ++ nativeBuildInputs; - - dontWrapGApps = true; - - preUnpack = '' - ${lib.optionalString (!autoDepsList) '' - if ! { [ '${lib.boolToString (depsListFile != null)}' = 'true' ] ${lib.optionalString (depsListFile != null) "&& cmp -s <(jq -Sc . '${depsListFile}') <(jq -Sc . '${finalAttrs.passthru.depsListFile}')"}; }; then - echo 1>&2 -e '\nThe dependency list file was either not given or differs from the expected result.' \ - '\nPlease choose one of the following solutions:' \ - '\n - Duplicate the following file and pass it to the depsListFile argument.' \ - '\n ${finalAttrs.passthru.depsListFile}' \ - '\n - Set autoDepsList to true (not supported by Hydra or permitted in Nixpkgs)'. - exit 1 - fi - ''} - - ${preUnpack} - ''; - - configurePhase = '' - runHook preConfigure - - ${flutterSetupScript} - - runHook postConfigure - ''; - - buildPhase = '' - runHook preBuild - - mkdir -p build/flutter_assets/fonts - - doPubGet flutter pub get --offline -v - flutter build linux -v --release --split-debug-info="$debug" ${builtins.concatStringsSep " " (map (flag: "\"${flag}\"") finalAttrs.flutterBuildFlags)} - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - built=build/linux/*/release/bundle - - mkdir -p $out/bin - mv $built $out/app - - for f in $(find $out/app -iname "*.desktop" -type f); do - install -D $f $out/share/applications/$(basename $f) - done - - for f in $(find $out/app -maxdepth 1 -type f); do - ln -s $f $out/bin/$(basename $f) - done - - # make *.so executable - find $out/app -iname "*.so" -type f -exec chmod +x {} + - - # remove stuff like /build/source/packages/ubuntu_desktop_installer/linux/flutter/ephemeral - for f in $(find $out/app -executable -type f); do - if patchelf --print-rpath "$f" | grep /build; then # this ignores static libs (e,g. libapp.so) also - echo "strip RPath of $f" - newrp=$(patchelf --print-rpath $f | sed -r "s|/build.*ephemeral:||g" | sed -r "s|/build.*profile:||g") - patchelf --set-rpath "$newrp" "$f" - fi - done - - runHook postInstall - ''; - - postFixup = '' - # Add runtime library dependencies to the LD_LIBRARY_PATH. - # For some reason, the RUNPATH of the executable is not used to load dynamic libraries in dart:ffi with DynamicLibrary.open(). - # - # This could alternatively be fixed with patchelf --add-needed, but this would cause all the libraries to be opened immediately, - # which is not what application authors expect. - for f in "$out"/bin/*; do - wrapProgram "$f" \ - --suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath finalAttrs.runtimeDependencies}' \ - ''${gappsWrapperArgs[@]} \ - ${extraWrapProgramArgs} - done - - ${postFixup} - ''; - - passthru = (args.passthru or {}) // { - inherit (deps) depsListFile; - }; - }); - - packageOverrideRepository = (callPackage ../../development/compilers/flutter/package-overrides { }) // customPackageOverrides; - productPackages = builtins.filter (package: package.kind != "dev") - (if autoDepsList - then lib.importJSON deps.depsListFile - else - if depsListFile == null - then [ ] - else lib.importJSON depsListFile); -in -builtins.foldl' - (prev: package: - if packageOverrideRepository ? ${package.name} - then - prev.overrideAttrs - (packageOverrideRepository.${package.name} { - inherit (package) - name - version - kind - source - dependencies; - }) - else prev) - baseDerivation - productPackages + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ wrapGAppsHook ]; + buildInputs = (args.buildInputs or [ ]) ++ [ glib ]; + + dontDartBuild = true; + buildPhase = args.buildPhase or '' + runHook preBuild + + mkdir -p build/flutter_assets/fonts + + doPubGet flutter pub get --offline -v + flutter build linux -v --release --split-debug-info="$debug" ${builtins.concatStringsSep " " (map (flag: "\"${flag}\"") flutterBuildFlags)} + + runHook postBuild + ''; + + dontDartInstall = true; + installPhase = args.installPhase or '' + runHook preInstall + + built=build/linux/*/release/bundle + + mkdir -p $out/bin + mv $built $out/app + + for f in $(find $out/app -iname "*.desktop" -type f); do + install -D $f $out/share/applications/$(basename $f) + done + + for f in $(find $out/app -maxdepth 1 -type f); do + ln -s $f $out/bin/$(basename $f) + done + + # make *.so executable + find $out/app -iname "*.so" -type f -exec chmod +x {} + + + # remove stuff like /build/source/packages/ubuntu_desktop_installer/linux/flutter/ephemeral + for f in $(find $out/app -executable -type f); do + if patchelf --print-rpath "$f" | grep /build; then # this ignores static libs (e,g. libapp.so) also + echo "strip RPath of $f" + newrp=$(patchelf --print-rpath $f | sed -r "s|/build.*ephemeral:||g" | sed -r "s|/build.*profile:||g") + patchelf --set-rpath "$newrp" "$f" + fi + done + + runHook postInstall + ''; + + dontWrapGApps = true; + extraWrapProgramArgs = '' + ''${gappsWrapperArgs[@]} \ + ${extraWrapProgramArgs} + ''; +}) diff --git a/pkgs/build-support/go/module.nix b/pkgs/build-support/go/module.nix index 09b43063fb9..d0fd8928c91 100644 --- a/pkgs/build-support/go/module.nix +++ b/pkgs/build-support/go/module.nix @@ -297,6 +297,7 @@ let } // meta; }); in +lib.warnIf (args' ? vendorSha256) "`vendorSha256` is deprecated. Use `vendorHash` instead" lib.warnIf (buildFlags != "" || buildFlagsArray != "") "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" package diff --git a/pkgs/build-support/kernel/make-initrd-ng.nix b/pkgs/build-support/kernel/make-initrd-ng.nix index 2418838176e..f2f7aaa6d1b 100644 --- a/pkgs/build-support/kernel/make-initrd-ng.nix +++ b/pkgs/build-support/kernel/make-initrd-ng.nix @@ -54,7 +54,7 @@ in # guess may not align with u-boot's nomenclature correctly, so it can # be overridden. # See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 for a list. -, uInitrdArch ? stdenvNoCC.hostPlatform.linuxArch +, uInitrdArch ? stdenvNoCC.hostPlatform.ubootArch # The name of the compression, as recognised by u-boot. # See https://gitlab.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L195-204 for a list. diff --git a/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock b/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock index 78ae4a01da6..8965b8793cc 100644 --- a/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock +++ b/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock @@ -3,12 +3,6 @@ version = 3 [[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] name = "eyre" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -20,9 +14,9 @@ dependencies = [ [[package]] name = "goblin" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91766b1121940d622933a13e20665857648681816089c9bc2075c4b75a6e4f6b" +checksum = "a7666983ed0dd8d21a6f6576ee00053ca0926fb281a5522577a4dbd0f1b54143" dependencies = [ "log", "plain", @@ -37,12 +31,9 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "make-initrd-ng" @@ -54,9 +45,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "plain" @@ -66,18 +57,18 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "proc-macro2" -version = "1.0.42" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.20" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -93,9 +84,9 @@ dependencies = [ [[package]] name = "scroll_derive" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", @@ -104,9 +95,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.98" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -115,6 +106,6 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.2" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/pkgs/build-support/kernel/make-initrd-ng/src/main.rs b/pkgs/build-support/kernel/make-initrd-ng/src/main.rs index 53096a84232..daa688976c6 100644 --- a/pkgs/build-support/kernel/make-initrd-ng/src/main.rs +++ b/pkgs/build-support/kernel/make-initrd-ng/src/main.rs @@ -195,7 +195,7 @@ fn handle_path( .wrap_err_with(|| format!("failed to resolve symlink of {:?}", source))?; // Create the link, then push its target to the queue - if !target.exists() { + if !target.exists() && !target.is_symlink() { unix::fs::symlink(&link_target, &target).wrap_err_with(|| { format!("failed to symlink {:?} to {:?}", link_target, target) })?; diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index 7cfc0e9f9c0..eecea8b7675 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -1,4 +1,10 @@ -{ lib, stdenv, fetchNpmDeps, buildPackages, nodejs }: +{ lib +, stdenv +, fetchNpmDeps +, buildPackages +, nodejs +, darwin +} @ topLevelArgs: { name ? "${args.pname}-${args.version}" , src ? null @@ -34,16 +40,16 @@ , npmPruneFlags ? npmInstallFlags # Value for npm `--workspace` flag and directory in which the files to be installed are found. , npmWorkspace ? null +, nodejs ? topLevelArgs.nodejs +, npmDeps ? fetchNpmDeps { + inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch; + name = "${name}-npm-deps"; + hash = npmDepsHash; +} , ... } @ args: let - npmDeps = fetchNpmDeps { - inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch; - name = "${name}-npm-deps"; - hash = npmDepsHash; - }; - # .override {} negates splicing, so we need to use buildPackages explicitly npmHooks = buildPackages.npmHooks.override { inherit nodejs; @@ -54,7 +60,9 @@ in stdenv.mkDerivation (args // { inherit npmDeps npmBuildScript; - nativeBuildInputs = nativeBuildInputs ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook ]; + nativeBuildInputs = nativeBuildInputs + ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ] + ++ lib.optionals stdenv.isDarwin [ darwin.cctools ]; buildInputs = buildInputs ++ [ nodejs ]; strictDeps = true; diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh index 64ddcbd567f..a2f77bdc6d4 100644 --- a/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh @@ -14,11 +14,21 @@ npmInstallHook() { local dest="$packageOut/$(dirname "$file")" mkdir -p "$dest" cp "${npmWorkspace-.}/$file" "$dest" - done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm pack --json --dry-run ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm pack --json --dry-run --loglevel=warn --no-foreground-scripts ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")") + # Based on code from Python's buildPythonPackage wrap.sh script, for + # supporting both the case when makeWrapperArgs is an array and a + # IFS-separated string. + # + # TODO: remove the string branch when __structuredAttrs are used. + if [[ "${makeWrapperArgs+defined}" == "defined" && "$(declare -p makeWrapperArgs)" =~ ^'declare -a makeWrapperArgs=' ]]; then + local -a user_args=("${makeWrapperArgs[@]}") + else + local -a user_args="(${makeWrapperArgs:-})" + fi while IFS=" " read -ra bin; do mkdir -p "$out/bin" - makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}" + makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}" "${user_args[@]}" done < <(@jq@ --raw-output '(.bin | type) as $typ | if $typ == "string" then .name + " " + .bin elif $typ == "object" then .bin | to_entries | map(.key + " " + .value) | join("\n") @@ -34,7 +44,16 @@ npmInstallHook() { if [ ! -d "$nodeModulesPath" ]; then if [ -z "${dontNpmPrune-}" ]; then - npm prune --omit=dev --no-save ${npmWorkspace+--workspace=$npmWorkspace} $npmPruneFlags "${npmPruneFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}" + if ! npm prune --omit=dev --no-save ${npmWorkspace+--workspace=$npmWorkspace} $npmPruneFlags "${npmPruneFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then + echo + echo + echo "ERROR: npm prune step failed" + echo + echo 'If npm tried to download additional dependencies above, try setting `dontNpmPrune = true`.' + echo + + exit 1 + fi fi find node_modules -maxdepth 1 -type d -empty -delete diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock index 482eb6c7bea..4f6e177fd9a 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock @@ -4,24 +4,24 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "async-channel" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", "event-listener", @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -59,9 +59,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "block-buffer" @@ -74,9 +74,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "castaway" @@ -86,9 +86,12 @@ checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -98,33 +101,23 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" dependencies = [ "crossbeam-utils", ] [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] [[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] name = "crossbeam-deque" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -184,9 +177,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.63+curl-8.1.2" +version = "0.4.67+curl-8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb0fef7046022a1e2ad67a004978f0e3cacb9e3123dc62ce768f92197b771dc" +checksum = "3cc35d066510b197a0f72de863736641539957628c8a42e70e27c66849e77c34" dependencies = [ "cc", "libc", @@ -194,7 +187,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "winapi", + "windows-sys", ] [[package]] @@ -209,9 +202,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "env_logger" @@ -228,26 +221,15 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] name = "event-listener" version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -263,6 +245,12 @@ dependencies = [ ] [[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -295,7 +283,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -327,9 +315,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" @@ -368,24 +356,13 @@ dependencies = [ ] [[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - -[[package]] name = "is-terminal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.2", + "rustix", "windows-sys", ] @@ -416,21 +393,21 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "libc", @@ -440,27 +417,21 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -472,16 +443,6 @@ dependencies = [ ] [[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] name = "once_cell" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -495,9 +456,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -507,9 +468,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +checksum = "e52c774a4c39359c1d1c52e43f73dd91a75a614652c825408eec30c95a9b2067" [[package]] name = "percent-encoding" @@ -519,18 +480,18 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", @@ -539,9 +500,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pkg-config" @@ -593,18 +554,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.29" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -641,9 +602,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -651,14 +612,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -672,53 +631,51 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", + "regex-automata", "regex-syntax", ] [[package]] -name = "regex-syntax" -version = "0.7.2" +name = "regex-automata" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] [[package]] -name = "rustix" -version = "0.37.22" +name = "regex-syntax" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8818fa822adcc98b18fedbb3632a6a33213c070556b5aa7c4c8cc21cff565c4c" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rustix" -version = "0.38.2" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys", "windows-sys", ] [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -740,24 +697,24 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.166" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.166" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -766,9 +723,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.99" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -777,9 +734,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -788,9 +745,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -799,9 +756,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -829,9 +786,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.23" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -840,23 +797,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ - "autocfg", "cfg-if", - "fastrand", + "fastrand 2.0.1", "redox_syscall", - "rustix 0.37.22", + "rustix", "windows-sys", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" dependencies = [ "winapi-util", ] @@ -921,9 +877,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -933,9 +889,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -948,9 +904,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -972,15 +928,15 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -1010,9 +966,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -1034,9 +990,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1049,42 +1005,42 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml index 41347b6c2cc..0f7735a6e82 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml @@ -6,17 +6,17 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.71" +anyhow = "1.0.75" backoff = "0.4.0" -base64 = "0.21.2" +base64 = "0.21.4" digest = "0.10.7" env_logger = "0.10.0" isahc = { version = "1.7.2", default_features = false } -rayon = "1.7.0" -serde = { version = "1.0.164", features = ["derive"] } -serde_json = "1.0.99" -sha1 = "0.10.5" -sha2 = "0.10.7" -tempfile = "3.6.0" -url = { version = "2.4.0", features = ["serde"] } -walkdir = "2.3.3" +rayon = "1.8.0" +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" +sha1 = "0.10.6" +sha2 = "0.10.8" +tempfile = "3.8.0" +url = { version = "2.4.1", features = ["serde"] } +walkdir = "2.4.0" diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs index b37652ffdf8..86e9120de02 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs @@ -4,7 +4,7 @@ use rayon::prelude::*; use serde_json::{Map, Value}; use std::{ fs, - io::{self, Read}, + io::Write, process::{Command, Stdio}, }; use tempfile::{tempdir, TempDir}; @@ -106,7 +106,7 @@ impl Package { let specifics = match get_hosted_git_url(&resolved)? { Some(hosted) => { - let mut body = util::get_url_with_retry(&hosted)?; + let body = util::get_url_body_with_retry(&hosted)?; let workdir = tempdir()?; @@ -120,7 +120,7 @@ impl Package { .stdin(Stdio::piped()) .spawn()?; - io::copy(&mut body, &mut cmd.stdin.take().unwrap())?; + cmd.stdin.take().unwrap().write_all(&body)?; let exit = cmd.wait()?; @@ -154,13 +154,7 @@ impl Package { pub fn tarball(&self) -> anyhow::Result<Vec<u8>> { match &self.specifics { - Specifics::Registry { .. } => { - let mut body = Vec::new(); - - util::get_url_with_retry(&self.url)?.read_to_end(&mut body)?; - - Ok(body) - } + Specifics::Registry { .. } => Ok(util::get_url_body_with_retry(&self.url)?), Specifics::Git { workdir } => Ok(Command::new("tar") .args([ "--sort=name", diff --git a/pkgs/build-support/node/fetch-npm-deps/src/util.rs b/pkgs/build-support/node/fetch-npm-deps/src/util.rs index 7a220f681c0..7dd928fdc43 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/util.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/util.rs @@ -4,7 +4,7 @@ use isahc::{ Body, Request, RequestExt, }; use serde_json::{Map, Value}; -use std::{env, path::Path}; +use std::{env, io::Read, path::Path}; use url::Url; pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { @@ -28,7 +28,7 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { if let Some(host) = url.host_str() { if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") { if let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens) { - if let Some(token) = tokens.get(host).and_then(|val| val.as_str()) { + if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) { request = request.header("Authorization", format!("Bearer {token}")); } } @@ -38,15 +38,23 @@ pub fn get_url(url: &Url) -> Result<Body, isahc::Error> { Ok(request.body(())?.send()?.into_body()) } -pub fn get_url_with_retry(url: &Url) -> Result<Body, isahc::Error> { +pub fn get_url_body_with_retry(url: &Url) -> Result<Vec<u8>, isahc::Error> { retry(ExponentialBackoff::default(), || { - get_url(url).map_err(|err| { - if err.is_network() || err.is_timeout() { - backoff::Error::transient(err) - } else { - backoff::Error::permanent(err) - } - }) + get_url(url) + .and_then(|mut body| { + let mut buf = Vec::new(); + + body.read_to_end(&mut buf)?; + + Ok(buf) + }) + .map_err(|err| { + if err.is_network() || err.is_timeout() { + backoff::Error::transient(err) + } else { + backoff::Error::permanent(err) + } + }) }) .map_err(|backoff_err| match backoff_err { backoff::Error::Permanent(err) diff --git a/pkgs/build-support/node/fetch-yarn-deps/fixup.js b/pkgs/build-support/node/fetch-yarn-deps/fixup.js index 8b91e7efa63..732e569aba7 100755 --- a/pkgs/build-support/node/fetch-yarn-deps/fixup.js +++ b/pkgs/build-support/node/fetch-yarn-deps/fixup.js @@ -21,6 +21,8 @@ const fixupYarnLock = async (lockContents, verbose) => { if (verbose) console.log(`Rewriting URL ${url} for dependency ${dep}`) pkg.resolved = urlToName(url) + if (hash) + pkg.resolved += `#${hash}` return [dep, pkg] }) diff --git a/pkgs/build-support/node/fetch-yarn-deps/index.js b/pkgs/build-support/node/fetch-yarn-deps/index.js index 04f47362b10..de2a09ee904 100755 --- a/pkgs/build-support/node/fetch-yarn-deps/index.js +++ b/pkgs/build-support/node/fetch-yarn-deps/index.js @@ -37,7 +37,9 @@ const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => { res.on('end', () => { file.close() const h = hash.read() - if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`)) + if (expectedHash === undefined){ + console.log(`Warning: lockfile url ${url} doesn't end in "#<hash>" to validate against. Downloaded file had hash ${h}.`); + } else if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h}`)) resolve() }) res.on('error', e => reject(e)) diff --git a/pkgs/build-support/ocaml/topkg.nix b/pkgs/build-support/ocaml/topkg.nix new file mode 100644 index 00000000000..73be5815e44 --- /dev/null +++ b/pkgs/build-support/ocaml/topkg.nix @@ -0,0 +1,28 @@ +{ lib, stdenv, fetchurl, ocaml, findlib, topkg, ocamlbuild, cmdliner, odoc, b0 +}: + +{ pname, version, nativeBuildInputs ? [ ], buildInputs ? [ ], ... }@args: + +lib.throwIf (args ? minimalOCamlVersion + && lib.versionOlder ocaml.version args.minimalOCamlVersion) +"${pname}-${version} is not available for OCaml ${ocaml.version}" + +stdenv.mkDerivation ({ + + dontAddStaticConfigureFlags = true; + configurePlatforms = [ ]; + strictDeps = true; + inherit (topkg) buildPhase installPhase; + +} // (builtins.removeAttrs args [ "minimalOCamlVersion" ]) // { + + name = "ocaml${ocaml.version}-${pname}-${version}"; + + nativeBuildInputs = [ ocaml findlib ocamlbuild topkg ] ++ nativeBuildInputs; + buildInputs = [ topkg ] ++ buildInputs; + + meta = (args.meta or { }) // { + platforms = args.meta.platforms or ocaml.meta.platforms; + }; + +}) diff --git a/pkgs/build-support/php/build-composer-project.nix b/pkgs/build-support/php/build-composer-project.nix index 6aecf434577..778aa35fa6a 100644 --- a/pkgs/build-support/php/build-composer-project.nix +++ b/pkgs/build-support/php/build-composer-project.nix @@ -13,10 +13,12 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; nativeBuildInputs = (previousAttrs.nativeBuildInputs or [ ]) ++ [ composer composer-local-repo-plugin + phpDrv phpDrv.composerHooks.composerInstallHook ]; @@ -53,6 +55,13 @@ let runHook postInstall ''; + doInstallCheck = previousAttrs.doInstallCheck or false; + installCheckPhase = previousAttrs.installCheckPhase or '' + runHook preCheckInstall + + runHook postCheckInstall + ''; + composerRepository = phpDrv.mkComposerRepository { inherit composer composer-local-repo-plugin; inherit (finalAttrs) patches pname src vendorHash version; @@ -61,8 +70,13 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; }; + COMPOSER_CACHE_DIR="/dev/null"; + COMPOSER_DISABLE_NETWORK="1"; + COMPOSER_MIRROR_PATH_REPOS="1"; + meta = previousAttrs.meta or { } // { platforms = lib.platforms.all; }; diff --git a/pkgs/build-support/php/build-composer-repository.nix b/pkgs/build-support/php/build-composer-repository.nix index 30b0b48de75..5b31f86e61c 100644 --- a/pkgs/build-support/php/build-composer-repository.nix +++ b/pkgs/build-support/php/build-composer-repository.nix @@ -32,6 +32,7 @@ let composerNoDev = previousAttrs.composerNoDev or true; composerNoPlugins = previousAttrs.composerNoPlugins or true; composerNoScripts = previousAttrs.composerNoScripts or true; + composerStrictValidation = previousAttrs.composerStrictValidation or true; name = "${previousAttrs.pname}-${previousAttrs.version}-composer-repository"; @@ -41,6 +42,7 @@ let nativeBuildInputs = (previousAttrs.nativeBuildInputs or [ ]) ++ [ composer composer-local-repo-plugin + phpDrv phpDrv.composerHooks.composerRepositoryHook ]; @@ -74,6 +76,13 @@ let runHook postInstall ''; + doInstallCheck = previousAttrs.doInstallCheck or false; + installCheckPhase = previousAttrs.installCheckPhase or '' + runHook preCheckInstall + + runHook postCheckInstall + ''; + COMPOSER_CACHE_DIR = "/dev/null"; COMPOSER_MIRROR_PATH_REPOS = "1"; COMPOSER_HTACCESS_PROTECT = "0"; diff --git a/pkgs/build-support/php/build-pecl.nix b/pkgs/build-support/php/build-pecl.nix index 389447e066f..6f38a668f3a 100644 --- a/pkgs/build-support/php/build-pecl.nix +++ b/pkgs/build-support/php/build-pecl.nix @@ -8,10 +8,9 @@ , nativeBuildInputs ? [ ] , postPhpize ? "" , makeFlags ? [ ] -, src ? fetchurl { +, src ? fetchurl ({ url = "https://pecl.php.net/get/${pname}-${version}.tgz"; - inherit (args) sha256; - } + } // lib.filterAttrs (attrName: _: lib.elem attrName [ "sha256" "hash" ]) args) , passthru ? { } , ... }@args: diff --git a/pkgs/build-support/php/hooks/composer-install-hook.sh b/pkgs/build-support/php/hooks/composer-install-hook.sh index 86d17d0f50f..6fe1c4e5f7d 100644 --- a/pkgs/build-support/php/hooks/composer-install-hook.sh +++ b/pkgs/build-support/php/hooks/composer-install-hook.sh @@ -22,13 +22,49 @@ composerInstallConfigureHook() { fi if [[ ! -f "composer.lock" ]]; then - echo "No composer.lock file found, consider adding one to your repository to ensure reproducible builds." + composer \ + --no-ansi \ + --no-install \ + --no-interaction \ + ${composerNoDev:+--no-dev} \ + ${composerNoPlugins:+--no-plugins} \ + ${composerNoScripts:+--no-scripts} \ + update + + mkdir -p $out + cp composer.lock $out/ + + echo + echo -e "\e[31mERROR: No composer.lock found\e[0m" + echo + echo -e '\e[31mNo composer.lock file found, consider adding one to your repository to ensure reproducible builds.\e[0m' + echo -e "\e[31mIn the meantime, a composer.lock file has been generated for you in $out/composer.lock\e[0m" + echo + echo -e '\e[31mTo fix the issue:\e[0m' + echo -e "\e[31m1. Copy the composer.lock file from $out/composer.lock to the project's source:\e[0m" + echo -e "\e[31m cp $out/composer.lock <path>\e[0m" + echo -e '\e[31m2. Add the composerLock attribute, pointing to the copied composer.lock file:\e[0m' + echo -e '\e[31m composerLock = ./composer.lock;\e[0m' + echo - if [[ -f "${composerRepository}/composer.lock" ]]; then - cp ${composerRepository}/composer.lock composer.lock - fi + exit 1 + fi + + echo "Validating consistency between composer.lock and ${composerRepository}/composer.lock" + if ! @cmp@ -s "composer.lock" "${composerRepository}/composer.lock"; then + echo + echo -e "\e[31mERROR: vendorHash is out of date\e[0m" + echo + echo -e "\e[31mcomposer.lock is not the same in $composerRepository\e[0m" + echo + echo -e "\e[31mTo fix the issue:\e[0m" + echo -e '\e[31m1. Set vendorHash to an empty string: `vendorHash = "";`\e[0m' + echo -e '\e[31m2. Build the derivation and wait for it to fail with a hash mismatch\e[0m' + echo -e '\e[31m3. Copy the "got: sha256-..." value back into the vendorHash field\e[0m' + echo -e '\e[31m You should have: vendorHash = "sha256-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX=";\e[0m' + echo - echo "Using an autogenerated composer.lock file." + exit 1 fi chmod +w composer.json composer.lock @@ -43,9 +79,9 @@ composerInstallBuildHook() { # because the file contains hardcoded nix store paths, we generate it here. composer-local-repo-plugin --no-ansi build-local-repo -m "${composerRepository}" . - # Remove all the repositories of type "composer" + # Remove all the repositories of type "composer" and "vcs" # from the composer.json file. - jq -r -c 'del(try .repositories[] | select(.type == "composer"))' composer.json | sponge composer.json + jq -r -c 'del(try .repositories[] | select(.type == "composer" or .type == "vcs"))' composer.json | sponge composer.json # Configure composer to disable packagist and avoid using the network. composer config repo.packagist false @@ -54,7 +90,6 @@ composerInstallBuildHook() { # Since the composer.json file has been modified in the previous step, the # composer.lock file needs to be updated. - COMPOSER_DISABLE_NETWORK=1 \ COMPOSER_ROOT_VERSION="${version}" \ composer \ --lock \ @@ -72,7 +107,26 @@ composerInstallBuildHook() { composerInstallCheckHook() { echo "Executing composerInstallCheckHook" - composer validate --no-ansi --no-interaction + if ! composer validate --strict --no-ansi --no-interaction --quiet; then + if [ ! -z "${composerStrictValidation-}" ]; then + echo + echo -e "\e[31mERROR: composer files validation failed\e[0m" + echo + echo -e '\e[31mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[31mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + exit 1 + else + echo + echo -e "\e[33mWARNING: composer files validation failed\e[0m" + echo + echo -e '\e[33mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[33mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' + echo + fi + fi echo "Finished composerInstallCheckHook" } @@ -84,10 +138,7 @@ composerInstallInstallHook() { # the autoloader. # The COMPOSER_ROOT_VERSION environment variable is needed only for # vimeo/psalm. - COMPOSER_CACHE_DIR=/dev/null \ - COMPOSER_DISABLE_NETWORK=1 \ COMPOSER_ROOT_VERSION="${version}" \ - COMPOSER_MIRROR_PATH_REPOS="1" \ composer \ --no-ansi \ --no-interaction \ diff --git a/pkgs/build-support/php/hooks/composer-repository-hook.sh b/pkgs/build-support/php/hooks/composer-repository-hook.sh index 057acf1fcc3..03783d9d639 100644 --- a/pkgs/build-support/php/hooks/composer-repository-hook.sh +++ b/pkgs/build-support/php/hooks/composer-repository-hook.sh @@ -3,6 +3,7 @@ declare version declare composerNoDev declare composerNoPlugins declare composerNoScripts +declare composerStrictValidation preConfigureHooks+=(composerRepositoryConfigureHook) preBuildHooks+=(composerRepositoryBuildHook) @@ -17,7 +18,7 @@ composerRepositoryConfigureHook() { fi if [[ ! -f "composer.lock" ]]; then - echo "No composer.lock file found, consider adding one to your repository to ensure reproducible builds." + COMPOSER_ROOT_VERSION="${version}" \ composer \ --no-ansi \ --no-install \ @@ -26,7 +27,24 @@ composerRepositoryConfigureHook() { ${composerNoPlugins:+--no-plugins} \ ${composerNoScripts:+--no-scripts} \ update - echo "Using an autogenerated composer.lock file." + + mkdir -p $out + cp composer.lock $out/ + + echo + echo -e "\e[31mERROR: No composer.lock found\e[0m" + echo + echo -e '\e[31mNo composer.lock file found, consider adding one to your repository to ensure reproducible builds.\e[0m' + echo -e "\e[31mIn the meantime, a composer.lock file has been generated for you in $out/composer.lock\e[0m" + echo + echo -e '\e[31mTo fix the issue:\e[0m' + echo -e "\e[31m1. Copy the composer.lock file from $out/composer.lock to the project's source:\e[0m" + echo -e "\e[31m cp $out/composer.lock <path>\e[0m" + echo -e '\e[31m2. Add the composerLock attribute, pointing to the copied composer.lock file:\e[0m' + echo -e '\e[31m composerLock = ./composer.lock;\e[0m' + echo + + exit 1 fi echo "Finished composerRepositoryConfigureHook" @@ -40,7 +58,6 @@ composerRepositoryBuildHook() { # Build the local composer repository # The command 'build-local-repo' is provided by the Composer plugin # nix-community/composer-local-repo-plugin. - COMPOSER_CACHE_DIR=/dev/null \ composer-local-repo-plugin --no-ansi build-local-repo ${composerNoDev:+--no-dev} -r repository echo "Finished composerRepositoryBuildHook" @@ -49,7 +66,26 @@ composerRepositoryBuildHook() { composerRepositoryCheckHook() { echo "Executing composerRepositoryCheckHook" - composer validate --no-ansi --no-interaction + if ! composer validate --strict --no-ansi --no-interaction --quiet; then + if [ ! -z "${composerStrictValidation-}" ]; then + echo + echo -e "\e[31mERROR: composer files validation failed\e[0m" + echo + echo -e '\e[31mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[31mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + exit 1 + else + echo + echo -e "\e[33mWARNING: composer files validation failed\e[0m" + echo + echo -e '\e[33mThe validation of the composer.json and composer.lock failed.\e[0m' + echo -e '\e[33mMake sure that the file composer.lock is consistent with composer.json.\e[0m' + echo + echo -e '\e[33mThis check is not blocking, but it is recommended to fix the issue.\e[0m' + echo + fi + fi echo "Finished composerRepositoryCheckHook" } @@ -61,8 +97,8 @@ composerRepositoryInstallHook() { cp -ar repository/. $out/ - # Copy the composer.lock files to the output directory, in case it has been - # autogenerated. + # Copy the composer.lock files to the output directory, to be able to validate consistency with + # the src composer.lock file where this fixed-output derivation is used cp composer.lock $out/ echo "Finished composerRepositoryInstallHook" diff --git a/pkgs/build-support/php/hooks/default.nix b/pkgs/build-support/php/hooks/default.nix index e7de98647c3..240ec640723 100644 --- a/pkgs/build-support/php/hooks/default.nix +++ b/pkgs/build-support/php/hooks/default.nix @@ -1,22 +1,29 @@ -{ makeSetupHook +{ lib +, makeSetupHook +, diffutils , jq , moreutils , makeBinaryWrapper -, php +, cacert +, buildPackages }: { composerRepositoryHook = makeSetupHook { name = "composer-repository-hook.sh"; - propagatedBuildInputs = [ jq moreutils php ]; + propagatedBuildInputs = [ jq moreutils cacert ]; substitutions = { }; } ./composer-repository-hook.sh; composerInstallHook = makeSetupHook { name = "composer-install-hook.sh"; - propagatedBuildInputs = [ jq makeBinaryWrapper moreutils php ]; - substitutions = { }; + propagatedBuildInputs = [ jq makeBinaryWrapper moreutils cacert ]; + substitutions = { + # Specify the stdenv's `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong `diff`. + cmp = "${lib.getBin buildPackages.diffutils}/bin/cmp"; + }; } ./composer-install-hook.sh; } diff --git a/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix b/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix index d81f30f4baa..48d05b7a000 100644 --- a/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix +++ b/pkgs/build-support/php/pkgs/composer-local-repo-plugin.nix @@ -1,7 +1,9 @@ -{ callPackage, stdenvNoCC, lib, fetchFromGitHub, makeBinaryWrapper }: +{ php, callPackage, stdenvNoCC, lib, fetchFromGitHub, makeBinaryWrapper }: let - composer = callPackage ./composer-phar.nix { }; + composer = callPackage ./composer-phar.nix { + inherit (php.packages.composer) version pharHash; + }; composerKeys = stdenvNoCC.mkDerivation (finalComposerKeysAttrs: { pname = "composer-keys"; @@ -69,7 +71,6 @@ stdenvNoCC.mkDerivation (finalAttrs: { composer global config --quiet minimum-stability dev composer global config --quiet prefer-stable true - composer global config --quiet autoloader-suffix "nixPredictableAutoloaderSuffix" composer global config --quiet apcu-autoloader false composer global config --quiet allow-plugins.nix-community/composer-local-repo-plugin true composer global config --quiet repo.packagist false diff --git a/pkgs/build-support/php/pkgs/composer-phar.nix b/pkgs/build-support/php/pkgs/composer-phar.nix index 3efd9098d6d..f281334ab2d 100644 --- a/pkgs/build-support/php/pkgs/composer-phar.nix +++ b/pkgs/build-support/php/pkgs/composer-phar.nix @@ -10,15 +10,17 @@ , stdenvNoCC , unzip , xz + , version + , pharHash }: stdenvNoCC.mkDerivation (finalAttrs: { pname = "composer-phar"; - version = "2.6.3"; + inherit version; src = fetchurl { url = "https://github.com/composer/composer/releases/download/${finalAttrs.version}/composer.phar"; - hash = "sha256-5Yo5DKwN9FzPWj2VrpT6I57e2LeQf6LI91LwIDBPybE="; + hash = pharHash; }; dontUnpack = true; diff --git a/pkgs/build-support/references-by-popularity/default.nix b/pkgs/build-support/references-by-popularity/default.nix index dfc25275f34..2171c622f00 100644 --- a/pkgs/build-support/references-by-popularity/default.nix +++ b/pkgs/build-support/references-by-popularity/default.nix @@ -6,11 +6,8 @@ path: runCommand "closure-paths" exportReferencesGraph.graph = path; __structuredAttrs = true; preferLocalBuild = true; - PATH = "${coreutils}/bin:${python3}/bin"; - builder = builtins.toFile "builder" - '' - . .attrs.sh - python3 ${./closure-graph.py} .attrs.json graph > ''${outputs[out]} - ''; - } - "" + nativeBuildInputs = [ coreutils python3 ]; +} +'' + python3 ${./closure-graph.py} "$NIX_ATTRS_JSON_FILE" graph > ''${outputs[out]} +'' diff --git a/pkgs/build-support/rust/build-rust-crate/build-crate.nix b/pkgs/build-support/rust/build-rust-crate/build-crate.nix index 37bf3ec26f7..e842b6a3f50 100644 --- a/pkgs/build-support/rust/build-rust-crate/build-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/build-crate.nix @@ -1,6 +1,5 @@ { lib, stdenv , mkRustcDepArgs, mkRustcFeatureArgs, needUnstableCLI -, rust }: { crateName, @@ -21,7 +20,7 @@ (mkRustcDepArgs dependencies crateRenames) (mkRustcFeatureArgs crateFeatures) ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--target" (rust.toRustTargetSpec stdenv.hostPlatform) + "--target" stdenv.hostPlatform.rust.rustcTargetSpec ] ++ lib.optionals (needUnstableCLI dependencies) [ "-Z" "unstable-options" ] ++ extraRustcOpts @@ -41,6 +40,7 @@ ); binRustcOpts = lib.concatStringsSep " " ( + [ "-C linker=${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc" ] ++ baseRustcOpts ); diff --git a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix index 60310f17874..a6f2528a82b 100644 --- a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: +{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: { build , buildDependencies @@ -124,8 +124,8 @@ in '' export CARGO_PKG_AUTHORS="${authors}" export CARGO_PKG_DESCRIPTION="${crateDescription}" - export CARGO_CFG_TARGET_ARCH=${rust.toTargetArch stdenv.hostPlatform} - export CARGO_CFG_TARGET_OS=${rust.toTargetOs stdenv.hostPlatform} + export CARGO_CFG_TARGET_ARCH=${stdenv.hostPlatform.rust.platform.arch} + export CARGO_CFG_TARGET_OS=${stdenv.hostPlatform.rust.platform.os} export CARGO_CFG_TARGET_FAMILY="unix" export CARGO_CFG_UNIX=1 export CARGO_CFG_TARGET_ENV="gnu" @@ -136,8 +136,8 @@ in '' export CARGO_MANIFEST_DIR=$(pwd) export DEBUG="${toString (!release)}" export OPT_LEVEL="${toString optLevel}" - export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}" - export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}" + export TARGET="${stdenv.hostPlatform.rust.rustcTargetSpec}" + export HOST="${stdenv.buildPlatform.rust.rustcTargetSpec}" export PROFILE=${if release then "release" else "debug"} export OUT_DIR=$(pwd)/target/build/${crateName}.out export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0} diff --git a/pkgs/build-support/rust/build-rust-crate/default.nix b/pkgs/build-support/rust/build-rust-crate/default.nix index f6079b7316d..d569142af66 100644 --- a/pkgs/build-support/rust/build-rust-crate/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/default.nix @@ -10,7 +10,6 @@ , fetchCrate , pkgsBuildBuild , rustc -, rust , cargo , jq , libiconv @@ -71,18 +70,14 @@ let inherit (import ./log.nix { inherit lib; }) noisily echo_colored; configureCrate = import ./configure-crate.nix { - inherit lib stdenv rust echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; + inherit lib stdenv echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; }; buildCrate = import ./build-crate.nix { - inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI rust; + inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI; }; installCrate = import ./install-crate.nix { inherit stdenv; }; - - # Allow access to the rust attribute set from inside buildRustCrate, which - # has a parameter that shadows the name. - rustAttrs = rust; in /* The overridable pkgs.buildRustCrate function. @@ -310,7 +305,7 @@ crate_: lib.makeOverridable depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies); hashedMetadata = builtins.hashString "sha256" (crateName + "-" + crateVersion + "___" + toString (mkRustcFeatureArgs crateFeatures) + - "___" + depsMetadata + "___" + rustAttrs.toRustTarget stdenv.hostPlatform); + "___" + depsMetadata + "___" + stdenv.hostPlatform.rust.rustcTarget); in lib.substring 0 10 hashedMetadata; @@ -353,6 +348,10 @@ crate_: lib.makeOverridable extraRustcOpts buildTests codegenUnits; }; dontStrip = !release; + + # We need to preserve metadata in .rlib, which might get stripped on macOS. See https://github.com/NixOS/nixpkgs/issues/218712 + stripExclude = [ "*.rlib" ]; + installPhase = installCrate crateName metadata buildTests; # depending on the test setting we are either producing something with bins @@ -362,6 +361,10 @@ crate_: lib.makeOverridable meta = { mainProgram = crateName; + badPlatforms = [ + # Rust is currently unable to target the n32 ABI + lib.systems.inspect.patterns.isMips64n32 + ]; }; } // extraDerivationAttrs ) diff --git a/pkgs/build-support/rust/build-rust-package/default.nix b/pkgs/build-support/rust/build-rust-package/default.nix index 9b2e62fc240..cf2ddbd084b 100644 --- a/pkgs/build-support/rust/build-rust-package/default.nix +++ b/pkgs/build-support/rust/build-rust-package/default.nix @@ -1,7 +1,6 @@ { lib , importCargoLock , fetchCargoTarball -, rust , stdenv , callPackage , cargoBuildHook @@ -62,7 +61,6 @@ assert cargoVendorDir == null && cargoLock == null -> !(args ? cargoSha256 && args.cargoSha256 != null) && !(args ? cargoHash && args.cargoHash != null) -> throw "cargoSha256, cargoHash, cargoVendorDir, or cargoLock must be set"; -assert buildType == "release" || buildType == "debug"; let @@ -79,18 +77,13 @@ let sha256 = args.cargoSha256; } // depsExtraArgs); - target = rust.toRustTargetSpec stdenv.hostPlatform; + target = stdenv.hostPlatform.rust.rustcTargetSpec; targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; - # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 - # the "${}" is needed to transform the path into a /nix/store path before baseNameOf - shortTarget = if targetIsJSON then - (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) - else target; - sysroot = callPackage ./sysroot { } { - inherit target shortTarget; + inherit target; + shortTarget = stdenv.hostPlatform.rust.cargoShortTarget; RUSTFLAGS = args.RUSTFLAGS or ""; originalCargoToml = src + /Cargo.toml; # profile info is later extracted }; @@ -162,10 +155,15 @@ stdenv.mkDerivation ((removeAttrs args [ "depsExtraArgs" "cargoUpdateHook" "carg # Platforms without host tools from # https://doc.rust-lang.org/nightly/rustc/platform-support.html "armv7a-darwin" - "armv5tel-linux" "armv7a-linux" "m68k-linux" "riscv32-linux" + "armv5tel-linux" "armv7a-linux" "m68k-linux" "mipsel-linux" + "mips64el-linux" "riscv32-linux" "armv6l-netbsd" "x86_64-redox" "wasm32-wasi" ]; + badPlatforms = [ + # Rust is currently unable to target the n32 ABI + lib.systems.inspect.patterns.isMips64n32 + ]; } // meta; }) diff --git a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix index a6d53056d9c..bb95b7bdc35 100644 --- a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix +++ b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, rustPlatform, buildPackages }: +{ lib, stdenv, rustPlatform, buildPackages }: { shortTarget, originalCargoToml, target, RUSTFLAGS }: @@ -26,7 +26,7 @@ in rustPlatform.buildRustPackage { done export RUST_SYSROOT=$(rustc --print=sysroot) - host=${rust.toRustTarget stdenv.buildPlatform} + host=${stdenv.buildPlatform.rust.rustcTarget} cp -r $RUST_SYSROOT/lib/rustlib/$host $out ''; diff --git a/pkgs/build-support/rust/hooks/cargo-build-hook.sh b/pkgs/build-support/rust/hooks/cargo-build-hook.sh index af94e02e38a..ed982c7ff30 100644 --- a/pkgs/build-support/rust/hooks/cargo-build-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-build-hook.sh @@ -17,7 +17,7 @@ cargoBuildHook() { fi if [ "${cargoBuildType}" != "debug" ]; then - cargoBuildProfileFlag="--${cargoBuildType}" + cargoBuildProfileFlag="--profile ${cargoBuildType}" fi if [ -n "${cargoBuildNoDefaultFeatures-}" ]; then @@ -30,13 +30,8 @@ cargoBuildHook() { ( set -x - env \ - "CC_@rustBuildPlatform@=@ccForBuild@" \ - "CXX_@rustBuildPlatform@=@cxxForBuild@" \ - "CC_@rustTargetPlatform@=@ccForHost@" \ - "CXX_@rustTargetPlatform@=@cxxForHost@" \ - cargo build -j $NIX_BUILD_CORES \ - --target @rustTargetPlatformSpec@ \ + @setEnv@ cargo build -j $NIX_BUILD_CORES \ + --target @rustHostPlatformSpec@ \ --frozen \ ${cargoBuildProfileFlag} \ ${cargoBuildNoDefaultFeaturesFlag} \ diff --git a/pkgs/build-support/rust/hooks/cargo-check-hook.sh b/pkgs/build-support/rust/hooks/cargo-check-hook.sh index 57fc2779cfe..971a140ec17 100644 --- a/pkgs/build-support/rust/hooks/cargo-check-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-check-hook.sh @@ -17,7 +17,7 @@ cargoCheckHook() { fi if [ "${cargoCheckType}" != "debug" ]; then - cargoCheckProfileFlag="--${cargoCheckType}" + cargoCheckProfileFlag="--profile ${cargoCheckType}" fi if [ -n "${cargoCheckNoDefaultFeatures-}" ]; then @@ -29,7 +29,7 @@ cargoCheckHook() { fi argstr="${cargoCheckProfileFlag} ${cargoCheckNoDefaultFeaturesFlag} ${cargoCheckFeaturesFlag} - --target @rustTargetPlatformSpec@ --frozen ${cargoTestFlags}" + --target @rustHostPlatformSpec@ --frozen ${cargoTestFlags}" ( set -x diff --git a/pkgs/build-support/rust/hooks/cargo-install-hook.sh b/pkgs/build-support/rust/hooks/cargo-install-hook.sh index 69ce7266936..24a6e6fa9eb 100644 --- a/pkgs/build-support/rust/hooks/cargo-install-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-install-hook.sh @@ -1,7 +1,7 @@ cargoInstallPostBuildHook() { echo "Executing cargoInstallPostBuildHook" - releaseDir=target/@shortTarget@/$cargoBuildType + releaseDir=target/@targetSubdirectory@/$cargoBuildType tmpDir="${releaseDir}-tmp"; mkdir -p $tmpDir @@ -21,7 +21,7 @@ cargoInstallHook() { # rename the output dir to a architecture independent one - releaseDir=target/@shortTarget@/$cargoBuildType + releaseDir=target/@targetSubdirectory@/$cargoBuildType tmpDir="${releaseDir}-tmp"; mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep "${tmpDir}$") diff --git a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh index de85683ead2..29ba18a6a1e 100644 --- a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh @@ -17,7 +17,7 @@ cargoNextestHook() { fi if [ "${cargoCheckType}" != "debug" ]; then - cargoCheckProfileFlag="--${cargoCheckType}" + cargoCheckProfileFlag="--cargo-profile ${cargoCheckType}" fi if [ -n "${cargoCheckNoDefaultFeatures-}" ]; then @@ -29,7 +29,7 @@ cargoNextestHook() { fi argstr="${cargoCheckProfileFlag} ${cargoCheckNoDefaultFeaturesFlag} ${cargoCheckFeaturesFlag} - --target @rustTargetPlatformSpec@ --frozen ${cargoTestFlags}" + --target @rustHostPlatformSpec@ --frozen ${cargoTestFlags}" ( set -x diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix index 2eb388fe07b..7703ff4abad 100644 --- a/pkgs/build-support/rust/hooks/default.nix +++ b/pkgs/build-support/rust/hooks/default.nix @@ -9,32 +9,20 @@ , rust , rustc , stdenv -, target ? rust.toRustTargetSpec stdenv.hostPlatform -}: -let - targetIsJSON = lib.hasSuffix ".json" target; +# This confusingly-named parameter indicates the *subdirectory of +# `target/` from which to copy the build artifacts. It is derived +# from a stdenv platform (or a JSON file). +, target ? stdenv.hostPlatform.rust.cargoShortTarget +}: - # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 - # the "${}" is needed to transform the path into a /nix/store path before baseNameOf - shortTarget = if targetIsJSON then - (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) - else target; - ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; - cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; - ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; - cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; - rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform; - rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform; - rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; -in { +{ cargoBuildHook = callPackage ({ }: makeSetupHook { name = "cargo-build-hook.sh"; propagatedBuildInputs = [ cargo ]; substitutions = { - inherit ccForBuild ccForHost cxxForBuild cxxForHost - rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec setEnv; }; } ./cargo-build-hook.sh) {}; @@ -43,7 +31,7 @@ in { name = "cargo-check-hook.sh"; propagatedBuildInputs = [ cargo ]; substitutions = { - inherit rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec; }; } ./cargo-check-hook.sh) {}; @@ -52,7 +40,7 @@ in { name = "cargo-install-hook.sh"; propagatedBuildInputs = [ ]; substitutions = { - inherit shortTarget; + targetSubdirectory = target; }; } ./cargo-install-hook.sh) {}; @@ -61,7 +49,7 @@ in { name = "cargo-nextest-hook.sh"; propagatedBuildInputs = [ cargo cargo-nextest ]; substitutions = { - inherit rustTargetPlatformSpec; + inherit (rust.envVars) rustHostPlatformSpec; }; } ./cargo-nextest-hook.sh) {}; @@ -77,24 +65,27 @@ in { diff = "${lib.getBin buildPackages.diffutils}/bin/diff"; cargoConfig = '' - [target."${rust.toRustTarget stdenv.buildPlatform}"] - "linker" = "${ccForBuild}" + [target."${stdenv.buildPlatform.rust.rustcTarget}"] + "linker" = "${rust.envVars.ccForBuild}" ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${shortTarget}"] - "linker" = "${ccForHost}" + [target."${stdenv.hostPlatform.rust.rustcTarget}"] + "linker" = "${rust.envVars.ccForHost}" ''} "rustflags" = [ "-C", "target-feature=${if stdenv.hostPlatform.isStatic then "+" else "-"}crt-static" ] ''; }; } ./cargo-setup-hook.sh) {}; - maturinBuildHook = callPackage ({ }: + maturinBuildHook = callPackage ({ pkgsHostTarget }: makeSetupHook { name = "maturin-build-hook.sh"; - propagatedBuildInputs = [ cargo maturin rustc ]; + propagatedBuildInputs = [ + pkgsHostTarget.maturin + pkgsHostTarget.cargo + pkgsHostTarget.rustc + ]; substitutions = { - inherit ccForBuild ccForHost cxxForBuild cxxForHost - rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + inherit (rust.envVars) rustTargetPlatformSpec setEnv; }; } ./maturin-build-hook.sh) {}; diff --git a/pkgs/build-support/rust/hooks/maturin-build-hook.sh b/pkgs/build-support/rust/hooks/maturin-build-hook.sh index 62d5619660c..d5ff069290b 100644 --- a/pkgs/build-support/rust/hooks/maturin-build-hook.sh +++ b/pkgs/build-support/rust/hooks/maturin-build-hook.sh @@ -9,12 +9,7 @@ maturinBuildHook() { ( set -x - env \ - "CC_@rustBuildPlatform@=@ccForBuild@" \ - "CXX_@rustBuildPlatform@=@cxxForBuild@" \ - "CC_@rustTargetPlatform@=@ccForHost@" \ - "CXX_@rustTargetPlatform@=@cxxForHost@" \ - maturin build \ + @setEnv@ maturin build \ --jobs=$NIX_BUILD_CORES \ --frozen \ --target @rustTargetPlatformSpec@ \ diff --git a/pkgs/build-support/rust/lib/default.nix b/pkgs/build-support/rust/lib/default.nix index aa5ba14c139..dad8ab52823 100644 --- a/pkgs/build-support/rust/lib/default.nix +++ b/pkgs/build-support/rust/lib/default.nix @@ -1,66 +1,81 @@ -{ lib }: +{ lib +, stdenv +, buildPackages +, targetPackages +}: rec { - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch - toTargetArch = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.arch - else if platform.isAarch32 then "arm" - else if platform.isMips64 then "mips64" # never add "el" suffix - else if platform.isPower64 then "powerpc64" # never add "le" suffix - else platform.parsed.cpu.name; + # These environment variables must be set when using `cargo-c` and + # several other tools which do not deal well with cross + # compilation. The symptom of the problem they fix is errors due + # to buildPlatform CFLAGS being passed to the + # hostPlatform-targeted compiler -- for example, `-m64` being + # passed on a build=x86_64/host=aarch64 compilation. + envVars = let + ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; + cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; + ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; + cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os - toTargetOs = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.os or "none" - else if platform.isDarwin then "macos" - else platform.parsed.kernel.name; + # Unfortunately we must use the dangerous `targetPackages` here + # because hooks are artificially phase-shifted one slot earlier + # (they go in nativeBuildInputs, so the hostPlatform looks like + # a targetPlatform to them). + ccForTarget = "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc"; + cxxForTarget = "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++"; - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family - toTargetFamily = platform: - if platform ? rustc.platform.target-family - then - ( - # Since https://github.com/rust-lang/rust/pull/84072 - # `target-family` is a list instead of single value. - let - f = platform.rustc.platform.target-family; - in - if builtins.isList f then f else [ f ] - ) - else lib.optional platform.isUnix "unix" - ++ lib.optional platform.isWindows "windows"; + rustBuildPlatform = stdenv.buildPlatform.rust.rustcTarget; + rustBuildPlatformSpec = stdenv.buildPlatform.rust.rustcTargetSpec; + rustHostPlatform = stdenv.hostPlatform.rust.rustcTarget; + rustHostPlatformSpec = stdenv.hostPlatform.rust.rustcTargetSpec; + rustTargetPlatform = stdenv.targetPlatform.rust.rustcTarget; + rustTargetPlatformSpec = stdenv.targetPlatform.rust.rustcTargetSpec; + in { + inherit + ccForBuild cxxForBuild rustBuildPlatform rustBuildPlatformSpec + ccForHost cxxForHost rustHostPlatform rustHostPlatformSpec + ccForTarget cxxForTarget rustTargetPlatform rustTargetPlatformSpec; - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor - toTargetVendor = platform: let - inherit (platform.parsed) vendor; - in platform.rustc.platform.vendor or { - "w64" = "pc"; - }.${vendor.name} or vendor.name; - - # Returns the name of the rust target, even if it is custom. Adjustments are - # because rust has slightly different naming conventions than we do. - toRustTarget = platform: let - inherit (platform.parsed) cpu kernel abi; - cpu_ = platform.rustc.platform.arch or { - "armv7a" = "armv7"; - "armv7l" = "armv7"; - "armv6l" = "arm"; - "armv5tel" = "armv5te"; - "riscv64" = "riscv64gc"; - }.${cpu.name} or cpu.name; - vendor_ = toTargetVendor platform; - in platform.rustc.config - or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}"; - - # Returns the name of the rust target if it is standard, or the json file - # containing the custom target spec. - toRustTargetSpec = platform: - if platform ? rustc.platform - then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform) - else toRustTarget platform; - - # Returns true if the target is no_std - # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421 - IsNoStdTarget = platform: let rustTarget = toRustTarget platform; in - builtins.any (t: lib.hasInfix t rustTarget) ["-none" "nvptx" "switch" "-uefi"]; + # Prefix this onto a command invocation in order to set the + # variables needed by cargo. + # + setEnv = '' + env \ + '' + # Due to a bug in how splicing and targetPackages works, in + # situations where targetPackages is irrelevant + # targetPackages.stdenv.cc is often simply wrong. We must omit + # the following lines when rustTargetPlatform collides with + # rustHostPlatform. + + lib.optionalString (rustTargetPlatform != rustHostPlatform) '' + "CC_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${ccForTarget}" \ + "CXX_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${cxxForTarget}" \ + "CARGO_TARGET_${stdenv.targetPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForTarget}" \ + '' + '' + "CC_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${ccForHost}" \ + "CXX_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${cxxForHost}" \ + "CARGO_TARGET_${stdenv.hostPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForHost}" \ + '' + '' + "CC_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${ccForBuild}" \ + "CXX_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${cxxForBuild}" \ + "CARGO_TARGET_${stdenv.buildPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForBuild}" \ + "CARGO_BUILD_TARGET=${rustBuildPlatform}" \ + "HOST_CC=${buildPackages.stdenv.cc}/bin/cc" \ + "HOST_CXX=${buildPackages.stdenv.cc}/bin/c++" \ + ''; + }; +} // lib.mapAttrs (old: new: platform: + # TODO: enable warning after 23.05 is EOL. + # lib.warn "`rust.${old} platform` is deprecated. Use `platform.rust.${new}` instead." + lib.getAttrFromPath new platform.rust) +{ + toTargetArch = [ "platform" "arch" ]; + toTargetOs = [ "platform" "os" ]; + toTargetFamily = [ "platform" "target-family" ]; + toTargetVendor = [ "platform" "vendor" ]; + toRustTarget = [ "rustcTarget" ]; + toRustTargetSpec = [ "rustcTargetSpec" ]; + toRustTargetSpecShort = [ "cargoShortTarget" ]; + toRustTargetForUseInEnvVars = [ "cargoEnvVarTarget" ]; + IsNoStdTarget = [ "isNoStdTarget" ]; } diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.py b/pkgs/build-support/setup-hooks/auto-patchelf.py index 965384b876f..261f5585480 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.py +++ b/pkgs/build-support/setup-hooks/auto-patchelf.py @@ -174,7 +174,7 @@ class Dependency: found: bool = False # Whether it was found somewhere -def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List[Path] = []) -> list[Dependency]: +def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List[Path] = [], extra_args: List[str] = []) -> list[Dependency]: try: with open_elf(path) as elf: @@ -213,7 +213,7 @@ def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List if file_is_dynamic_executable: print("setting interpreter of", path) subprocess.run( - ["patchelf", "--set-interpreter", interpreter_path.as_posix(), path.as_posix()], + ["patchelf", "--set-interpreter", interpreter_path.as_posix(), path.as_posix()] + extra_args, check=True) rpath += runtime_deps @@ -250,7 +250,7 @@ def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: List if rpath: print("setting RPATH to:", rpath_str) subprocess.run( - ["patchelf", "--set-rpath", rpath_str, path.as_posix()], + ["patchelf", "--set-rpath", rpath_str, path.as_posix()] + extra_args, check=True) return dependencies @@ -262,7 +262,8 @@ def auto_patchelf( runtime_deps: List[Path], recursive: bool = True, ignore_missing: List[str] = [], - append_rpaths: List[Path] = []) -> None: + append_rpaths: List[Path] = [], + extra_args: List[str] = []) -> None: if not paths_to_patch: sys.exit("No paths to patch, stopping.") @@ -275,7 +276,7 @@ def auto_patchelf( dependencies = [] for path in chain.from_iterable(glob(p, '*', recursive) for p in paths_to_patch): if not path.is_symlink() and path.is_file(): - dependencies += auto_patchelf_file(path, runtime_deps, append_rpaths) + dependencies += auto_patchelf_file(path, runtime_deps, append_rpaths, extra_args) missing = [dep for dep in dependencies if not dep.found] @@ -333,6 +334,12 @@ def main() -> None: type=Path, help="Paths to append to all runtime paths unconditionally", ) + parser.add_argument( + "--extra-args", + nargs="*", + type=str, + help="Extra arguments to pass to patchelf" + ) print("automatically fixing dependencies for ELF files") args = parser.parse_args() @@ -344,7 +351,8 @@ def main() -> None: args.runtime_dependencies, args.recursive, args.ignore_missing, - append_rpaths=args.append_rpaths) + append_rpaths=args.append_rpaths, + extra_args=args.extra_args) interpreter_path: Path = None # type: ignore diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 0625565606f..371389df427 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -63,6 +63,7 @@ autoPatchelf() { local appendRunpathsArray=($appendRunpaths) local runtimeDependenciesArray=($runtimeDependencies) + local patchelfFlagsArray=($patchelfFlags) @pythonInterpreter@ @autoPatchelfScript@ \ ${norecurse:+--no-recurse} \ --ignore-missing "${ignoreMissingDepsArray[@]}" \ @@ -70,7 +71,8 @@ autoPatchelf() { --libs "${autoPatchelfLibs[@]}" \ "${extraAutoPatchelfLibs[@]}" \ --runtime-dependencies "${runtimeDependenciesArray[@]/%//lib}" \ - --append-rpaths "${appendRunpathsArray[@]}" + --append-rpaths "${appendRunpathsArray[@]}" \ + --extra-args "${patchelfFlagsArray[@]}" } # XXX: This should ultimately use fixupOutputHooks but we currently don't have diff --git a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh index 55e196e654d..e103fe77d9b 100644 --- a/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh +++ b/pkgs/build-support/setup-hooks/fix-darwin-dylib-names.sh @@ -36,5 +36,5 @@ fixDarwinDylibNames() { fixDarwinDylibNamesIn() { local dir="$1" - fixDarwinDylibNames $(find "$dir" -name "*.dylib") + fixDarwinDylibNames $(find "$dir" -name "*.dylib" -o -name "*.so" -o -name "*.so.*") } diff --git a/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh b/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh index 88a50befd73..6cd01f6bf63 100644 --- a/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh +++ b/pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh @@ -193,8 +193,23 @@ makeCWrapper() { addFlags() { local n flag before after var + + # Disable file globbing, since bash will otherwise try to find + # filenames matching the the value to be prefixed/suffixed if + # it contains characters considered wildcards, such as `?` and + # `*`. We want the value as is, except we also want to split + # it on on the separator; hence we can't quote it. + local reenableGlob=0 + if [[ ! -o noglob ]]; then + reenableGlob=1 + fi + set -o noglob # shellcheck disable=SC2086 before=($1) after=($2) + if (( reenableGlob )); then + set +o noglob + fi + var="argv_tmp" printf '%s\n' "char **$var = calloc(${#before[@]} + argc + ${#after[@]} + 1, sizeof(*$var));" printf '%s\n' "assert($var != NULL);" diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index 8b8732af065..744c7807f46 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck, haskell }: +{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }: let inherit (lib) @@ -311,6 +311,8 @@ rec { Similar to writeShellScriptBin and writeScriptBin. Writes an executable Shell script to /nix/store/<store path>/bin/<name> and checks its syntax with shellcheck and the shell's -n option. + Individual checks can be foregone by putting them in the excludeShellChecks + list, e.g. [ "SC2016" ]. Automatically includes sane set of shellopts (errexit, nounset, pipefail) and handles creation of PATH based on runtimeInputs @@ -338,6 +340,7 @@ rec { , runtimeInputs ? [ ] , meta ? { } , checkPhase ? null + , excludeShellChecks ? [ ] }: writeTextFile { inherit name meta; @@ -362,11 +365,12 @@ rec { # GHC (=> shellcheck) isn't supported on some platforms (such as risc-v) # but we still want to use writeShellApplication on those platforms let - shellcheckSupported = lib.meta.availableOn stdenv.buildPlatform shellcheck.compiler; + shellcheckSupported = lib.meta.availableOn stdenv.buildPlatform shellcheck-minimal.compiler; + excludeOption = lib.optionalString (excludeShellChecks != [ ]) "--exclude '${lib.concatStringsSep "," excludeShellChecks}'"; shellcheckCommand = lib.optionalString shellcheckSupported '' # use shellcheck which does not include docs # pandoc takes long to build and documentation isn't needed for just running the cli - ${lib.getExe (haskell.lib.compose.justStaticExecutables shellcheck.unwrapped)} "$target" + ${lib.getExe shellcheck-minimal} ${excludeOption} "$target" ''; in if checkPhase == null then '' diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix index 683f4b9fd04..cbd1b388ef6 100644 --- a/pkgs/build-support/trivial-builders/test/default.nix +++ b/pkgs/build-support/trivial-builders/test/default.nix @@ -25,6 +25,7 @@ recurseIntoAttrs { then callPackage ./references.nix {} else null; writeCBin = callPackage ./writeCBin.nix {}; + writeShellApplication = callPackage ./writeShellApplication.nix {}; writeScriptBin = callPackage ./writeScriptBin.nix {}; writeShellScript = callPackage ./write-shell-script.nix {}; writeShellScriptBin = callPackage ./writeShellScriptBin.nix {}; diff --git a/pkgs/build-support/trivial-builders/test/writeShellApplication.nix b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix new file mode 100644 index 00000000000..6ce6f0720fc --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/writeShellApplication.nix @@ -0,0 +1,29 @@ +/* + Run with: + + cd nixpkgs + nix-build -A tests.trivial-builders.writeShellApplication +*/ + +{ lib, writeShellApplication, runCommand }: +let + pkg = writeShellApplication { + name = "test-script"; + excludeShellChecks = [ "SC2016" ]; + text = '' + echo -e '#!/usr/bin/env bash\n' \ + 'echo "$SHELL"' > /tmp/something.sh # this line would normally + # ...cause shellcheck error + ''; + }; +in + assert pkg.meta.mainProgram == "test-script"; + runCommand "test-writeShellApplication" { } '' + + echo Testing if writeShellApplication builds without shellcheck error... + + target=${lib.getExe pkg} + + touch $out + '' + diff --git a/pkgs/build-support/writers/data.nix b/pkgs/build-support/writers/data.nix index 48f9bc547ed..45ed5360eae 100644 --- a/pkgs/build-support/writers/data.nix +++ b/pkgs/build-support/writers/data.nix @@ -1,4 +1,4 @@ -{ lib, runCommand, dasel }: +{ lib, pkgs, formats, runCommand, dasel }: let daselBin = lib.getExe dasel; @@ -23,7 +23,7 @@ rec { # writeJSON = makeDataWriter { input = builtins.toJSON; output = "cp $inputPath $out"; }; # myConfig = writeJSON "config.json" { hello = "world"; } # - makeDataWriter = { input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data: + makeDataWriter = lib.warn "pkgs.writers.makeDataWriter is deprecated. Use pkgs.writeTextFile." ({ input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data: assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); let name = last (builtins.split "/" nameOrPath); @@ -40,41 +40,25 @@ rec { mkdir -p $out/$(dirname "${nameOrPath}") mv tmp $out/${nameOrPath} ''} - ''; + ''); - # Writes the content to text. - # - # Example: - # writeText "filename.txt" "file content" - writeText = makeDataWriter { - input = toString; - output = "cp $inputPath $out"; - }; + inherit (pkgs) writeText; # Writes the content to a JSON file. # # Example: # writeJSON "data.json" { hello = "world"; } - writeJSON = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w json > $out"; - }; + writeJSON = (pkgs.formats.json {}).generate; # Writes the content to a TOML file. # # Example: # writeTOML "data.toml" { hello = "world"; } - writeTOML = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w toml > $out"; - }; + writeTOML = (pkgs.formats.toml {}).generate; # Writes the content to a YAML file. # # Example: # writeYAML "data.yaml" { hello = "world"; } - writeYAML = makeDataWriter { - input = builtins.toJSON; - output = "${daselBin} -f $inputPath -r json -w yaml > $out"; - }; + writeYAML = (pkgs.formats.yaml {}).generate; } diff --git a/pkgs/build-support/writers/test.nix b/pkgs/build-support/writers/test.nix index 2411f8c03a7..005daf0be5b 100644 --- a/pkgs/build-support/writers/test.nix +++ b/pkgs/build-support/writers/test.nix @@ -7,6 +7,7 @@ , python3Packages , pypy3Packages , runCommand +, testers , writers , writeText }: @@ -36,14 +37,7 @@ let let expectedFile = writeText "${file.name}-expected" expected; in - runCommand "run-${file.name}" {} '' - if ! diff -u ${file} ${expectedFile}; then - echo 'test ${file.name} failed' - exit 1 - fi - - touch $out - ''; + testers.testEqualContents { expected = expectedFile; actual = file; assertion = "${file.name} matches"; }; in lib.recurseIntoAttrs { bin = lib.recurseIntoAttrs { @@ -261,7 +255,9 @@ lib.recurseIntoAttrs { toml = expectDataEqual { file = writeTOML "data.toml" { hello = "world"; }; - expected = "hello = 'world'\n"; + expected = '' + hello = "world" + ''; }; yaml = expectDataEqual { |