diff options
Diffstat (limited to 'pkgs/test')
37 files changed, 962 insertions, 446 deletions
diff --git a/pkgs/test/cc-wrapper/atomics.cc b/pkgs/test/cc-wrapper/atomics.cc new file mode 100644 index 00000000000..23601ae92f0 --- /dev/null +++ b/pkgs/test/cc-wrapper/atomics.cc @@ -0,0 +1,8 @@ +#include <atomic> +#include <cstdint> + +int main() +{ + std::atomic_int x = {0}; + return !std::atomic_is_lock_free(&x); +} diff --git a/pkgs/test/cc-wrapper/default.nix b/pkgs/test/cc-wrapper/default.nix index 8809030989e..a0088751d4a 100644 --- a/pkgs/test/cc-wrapper/default.nix +++ b/pkgs/test/cc-wrapper/default.nix @@ -9,7 +9,8 @@ let ); staticLibc = lib.optionalString (stdenv.hostPlatform.libc == "glibc") "-L ${glibc.static}/lib"; emulator = stdenv.hostPlatform.emulator buildPackages; - libcxxStdenvSuffix = lib.optionalString (stdenv.cc.libcxx != null) "-libcxx"; + isCxx = stdenv.cc.libcxx != null; + libcxxStdenvSuffix = lib.optionalString isCxx "-libcxx"; in stdenv.mkDerivation { pname = "cc-wrapper-test-${stdenv.cc.cc.pname}${libcxxStdenvSuffix}"; version = stdenv.cc.version; @@ -30,6 +31,21 @@ in stdenv.mkDerivation { $CXX -o cxx-check ${./cxx-main.cc} ${emulator} ./cxx-check + # test for https://github.com/NixOS/nixpkgs/issues/214524#issuecomment-1431745905 + # .../include/cxxabi.h:20:10: fatal error: '__cxxabi_config.h' file not found + # in libcxxStdenv + echo "checking whether cxxabi.h can be included... " >&2 + $CXX -o include-cxxabi ${./include-cxxabi.cc} + ${emulator} ./include-cxxabi + + # cxx doesn't have libatomic.so + ${lib.optionalString (!isCxx) '' + # https://github.com/NixOS/nixpkgs/issues/91285 + echo "checking whether libatomic.so can be linked... " >&2 + $CXX -shared -o atomics.so ${./atomics.cc} -latomic ${lib.optionalString (stdenv.cc.isClang && lib.versionOlder stdenv.cc.version "6.0.0" ) "-std=c++17"} + $READELF -d ./atomics.so | grep libatomic.so && echo "ok" >&2 || echo "failed" >&2 + ''} + ${lib.optionalString (stdenv.isDarwin && stdenv.cc.isClang) '' echo "checking whether compiler can build with CoreFoundation.framework... " >&2 mkdir -p foo/lib diff --git a/pkgs/test/cc-wrapper/include-cxxabi.cc b/pkgs/test/cc-wrapper/include-cxxabi.cc new file mode 100644 index 00000000000..6ffc97e414a --- /dev/null +++ b/pkgs/test/cc-wrapper/include-cxxabi.cc @@ -0,0 +1,8 @@ +#include <cxxabi.h> +#include <iostream> + +int main(int argc, char **argv) +{ + std::cerr << "ok" << std::endl; + return 0; +} diff --git a/pkgs/test/cross/default.nix b/pkgs/test/cross/default.nix index ff83aedca12..b4da2de5c5b 100644 --- a/pkgs/test/cross/default.nix +++ b/pkgs/test/cross/default.nix @@ -13,7 +13,7 @@ let compareTest = { emulator, pkgFun, hostPkgs, crossPkgs, exec, args ? [] }: let pkgName = (pkgFun hostPkgs).name; args' = lib.concatStringsSep " " args; - in crossPkgs.runCommand "test-${pkgName}-${crossPkgs.hostPlatform.config}" { + in crossPkgs.runCommand "test-${pkgName}-${crossPkgs.stdenv.hostPlatform.config}" { nativeBuildInputs = [ pkgs.dos2unix ]; } '' # Just in case we are using wine, get rid of that annoying extra @@ -59,12 +59,12 @@ let crossSystem = crossSystemFun system; }; - emulator = crossPkgs.hostPlatform.emulator pkgs; + emulator = crossPkgs.stdenv.hostPlatform.emulator pkgs; # Apply some transformation on windows to get dlls in the right # place. Unfortunately mingw doesn’t seem to be able to do linking # properly. - platformFun = pkg: if crossPkgs.hostPlatform.isWindows then + platformFun = pkg: if crossPkgs.stdenv.hostPlatform.isWindows then pkgs.buildEnv { name = "${pkg.name}-winlinks"; paths = [pkg] ++ pkg.buildInputs; @@ -92,7 +92,7 @@ let }; pkg-config = {platformFun, crossPkgs, emulator}: crossPkgs.runCommand - "test-pkg-config-${crossPkgs.hostPlatform.config}" + "test-pkg-config-${crossPkgs.stdenv.hostPlatform.config}" { depsBuildBuild = [ crossPkgs.pkgsBuildBuild.pkg-config ]; nativeBuildInputs = [ crossPkgs.pkgsBuildHost.pkg-config crossPkgs.buildPackages.zlib ]; @@ -134,6 +134,7 @@ let pkgs.pkgsMusl.stdenv pkgs.pkgsLLVM.stdenv pkgs.pkgsStatic.bash + #pkgs.pkgsCross.gnu64_simplekernel.bash # https://github.com/NixOS/nixpkgs/issues/264989 pkgs.pkgsCross.arm-embedded.stdenv pkgs.pkgsCross.sheevaplug.stdenv # for armv5tel pkgs.pkgsCross.raspberryPi.stdenv # for armv6l @@ -141,9 +142,23 @@ let pkgs.pkgsCross.m68k.stdenv pkgs.pkgsCross.aarch64-multiplatform.pkgsBuildTarget.gcc pkgs.pkgsCross.powernv.pkgsBuildTarget.gcc + pkgs.pkgsCross.s390.stdenv pkgs.pkgsCross.mips64el-linux-gnuabi64.stdenv pkgs.pkgsCross.mips64el-linux-gnuabin32.stdenv pkgs.pkgsCross.mingwW64.stdenv + + ] ++ lib.optionals (with pkgs.stdenv.buildPlatform; isx86_64 && isLinux) [ + # Musl-to-glibc cross on the same architecture tends to turn up + # lots of interesting corner cases. Only expected to work for + # x86_64-linux buildPlatform. + pkgs.pkgsMusl.pkgsCross.gnu64.hello + + # Two web browsers -- exercises almost the entire packageset + pkgs.pkgsCross.aarch64-multiplatform.qt5.qutebrowser + pkgs.pkgsCross.aarch64-multiplatform.firefox + + # Uses pkgsCross.riscv64-embedded; see https://github.com/NixOS/nixpkgs/issues/267859 + pkgs.spike ]; in { diff --git a/pkgs/test/cuda/cuda-samples/extension.nix b/pkgs/test/cuda/cuda-samples/extension.nix index 05861ee5e0e..664349416b7 100644 --- a/pkgs/test/cuda/cuda-samples/extension.nix +++ b/pkgs/test/cuda/cuda-samples/extension.nix @@ -11,13 +11,15 @@ final: prev: let "11.4" = "082dkk5y34wyvjgj2p5j1d00rk8xaxb9z0mhvz16bd469r1bw2qk"; "11.5" = "sha256-AKRZbke0K59lakhTi8dX2cR2aBuWPZkiQxyKaZTvHrI="; "11.6" = "sha256-AsLNmAplfuQbXg9zt09tXAuFJ524EtTYsQuUlV1tPkE="; - "11.7" = throw "The tag 11.7 of cuda-samples does not exist"; + # The tag 11.7 of cuda-samples does not exist "11.8" = "sha256-7+1P8+wqTKUGbCUBXGMDO9PkxYr2+PLDx9W2hXtXbuc="; "12.0" = "sha256-Lj2kbdVFrJo5xPYPMiE4BS7Z8gpU5JLKXVJhZABUe/g="; - }.${prev.cudaVersion}; + "12.1" = "sha256-xE0luOMq46zVsIEWwK4xjLs7NorcTIi9gbfZPVjIlqo="; + "12.2" = "sha256-pOy0qfDjA/Nr0T9PNKKefK/63gQnJV2MQsN2g3S2yng="; + }; -in { +in prev.lib.attrsets.optionalAttrs (builtins.hasAttr prev.cudaVersion sha256) { cuda-samples = final.callPackage ./generic.nix { - inherit sha256; + sha256 = sha256.${prev.cudaVersion}; }; } diff --git a/pkgs/test/cuda/cuda-samples/generic.nix b/pkgs/test/cuda/cuda-samples/generic.nix index 267eca10d8e..e690f32959f 100644 --- a/pkgs/test/cuda/cuda-samples/generic.nix +++ b/pkgs/test/cuda/cuda-samples/generic.nix @@ -1,31 +1,46 @@ -{ lib -, cudaPackages +{ autoAddOpenGLRunpathHook +, backendStdenv +, cmake +, cudatoolkit +, cudaVersion , fetchFromGitHub , fetchpatch -, addOpenGLRunpath -, cudatoolkit +, freeimage +, glfw3 +, lib , pkg-config , sha256 -, glfw3 -, freeimage }: -cudaPackages.backendStdenv.mkDerivation rec { +backendStdenv.mkDerivation (finalAttrs: { pname = "cuda-samples"; - version = lib.versions.majorMinor cudatoolkit.version; + version = cudaVersion; src = fetchFromGitHub { owner = "NVIDIA"; - repo = pname; - rev = "v${version}"; + repo = finalAttrs.pname; + rev = "v${finalAttrs.version}"; inherit sha256; }; - nativeBuildInputs = [ pkg-config addOpenGLRunpath glfw3 freeimage ]; + nativeBuildInputs = [ + pkg-config + autoAddOpenGLRunpathHook + glfw3 + freeimage + ] + # CMake has to run as a native, build-time dependency for libNVVM samples. + ++ lib.lists.optionals (lib.strings.versionAtLeast finalAttrs.version "12.2") [ + cmake + ]; + + # CMake is not the primary build tool -- that's still make. + # As such, we disable CMake's build system. + dontUseCmakeConfigure = true; buildInputs = [ cudatoolkit ]; # See https://github.com/NVIDIA/cuda-samples/issues/75. - patches = lib.optionals (version == "11.3") [ + patches = lib.optionals (finalAttrs.version == "11.3") [ (fetchpatch { url = "https://github.com/NVIDIA/cuda-samples/commit/5c3ec60faeb7a3c4ad9372c99114d7bb922fda8d.patch"; sha256 = "sha256-0XxdmNK9MPpHwv8+qECJTvXGlFxc+fIbta4ynYprfpU="; @@ -41,21 +56,15 @@ cudaPackages.backendStdenv.mkDerivation rec { installPhase = '' runHook preInstall - install -Dm755 -t $out/bin bin/${cudaPackages.backendStdenv.hostPlatform.parsed.cpu.name}/${cudaPackages.backendStdenv.hostPlatform.parsed.kernel.name}/release/* + install -Dm755 -t $out/bin bin/${backendStdenv.hostPlatform.parsed.cpu.name}/${backendStdenv.hostPlatform.parsed.kernel.name}/release/* runHook postInstall ''; - postFixup = '' - for exe in $out/bin/*; do - addOpenGLRunpath $exe - done - ''; - meta = { description = "Samples for CUDA Developers which demonstrates features in CUDA Toolkit"; # CUDA itself is proprietary, but these sample apps are not. license = lib.licenses.bsd3; - maintainers = with lib.maintainers; [ obsidian-systems-maintenance ]; + maintainers = with lib.maintainers; [ obsidian-systems-maintenance ] ++ lib.teams.cuda.members; }; -} +}) diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 05d8ee61e9a..9543e165a80 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -8,6 +8,8 @@ with pkgs; llvmTests = let pkgSets = lib.pipe pkgNames [ (filter (lib.hasPrefix "llvmPackages")) + (filter (n: n != "rocmPackages.llvm")) + # Is a throw alias. (filter (n: n != "llvmPackages_rocm")) (filter (n: n != "llvmPackages_latest")) (filter (n: n != "llvmPackages_git")) @@ -37,41 +39,42 @@ with pkgs; name = "cc-wrapper-supported"; builtGCC = let - names = lib.pipe (attrNames gccTests) ([ - (filter (n: lib.meta.availableOn stdenv.hostPlatform pkgs.${n}.cc)) + inherit (lib) filterAttrs; + sets = lib.pipe gccTests ([ + (filterAttrs (_: v: lib.meta.availableOn stdenv.hostPlatform v.stdenv.cc)) # Broken - (filter (n: n != "gcc49Stdenv")) - (filter (n: n != "gccMultiStdenv")) + (filterAttrs (n: _: n != "gcc49Stdenv")) + (filterAttrs (n: _: n != "gccMultiStdenv")) ] ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [ # fails with things like # ld: warning: ld: warning: object file (trunctfsf2_s.o) was built for newer macOS version (11.0) than being linked (10.5) # ld: warning: ld: warning: could not create compact unwind for ___fixunstfdi: register 20 saved somewhere other than in frame - (filter (n: n != "gcc11Stdenv")) + (filterAttrs (n: _: n != "gcc11Stdenv")) ]); in - toJSON (lib.genAttrs names (name: { name = pkgs.${name}; })); + toJSON sets; builtLLVM = let - names = lib.pipe (attrNames llvmTests) ([ - (filter (n: lib.meta.availableOn stdenv.hostPlatform pkgs.${n}.stdenv.cc)) - (filter (n: lib.meta.availableOn stdenv.hostPlatform pkgs.${n}.libcxxStdenv.cc)) + inherit (lib) filterAttrs; + sets = lib.pipe llvmTests ([ + (filterAttrs (_: v: lib.meta.availableOn stdenv.hostPlatform v.clang.stdenv.cc)) + (filterAttrs (_: v: lib.meta.availableOn stdenv.hostPlatform v.libcxx.stdenv.cc)) # libcxxStdenv broken # fix in https://github.com/NixOS/nixpkgs/pull/216273 ] ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ # libcxx does not build for some reason on aarch64-linux - (filter (n: n != "llvmPackages_7")) + (filterAttrs (n: _: n != "llvmPackages_7")) ] ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [ - (filter (n: n != "llvmPackages_5")) - (filter (n: n != "llvmPackages_6")) - (filter (n: n != "llvmPackages_7")) - (filter (n: n != "llvmPackages_8")) - (filter (n: n != "llvmPackages_9")) - (filter (n: n != "llvmPackages_10")) + (filterAttrs (n: _: n != "llvmPackages_6")) + (filterAttrs (n: _: n != "llvmPackages_7")) + (filterAttrs (n: _: n != "llvmPackages_8")) + (filterAttrs (n: _: n != "llvmPackages_9")) + (filterAttrs (n: _: n != "llvmPackages_10")) ]); in - toJSON (lib.genAttrs names (name: { stdenv = pkgs.${name}.stdenv; libcxx = pkgs.${name}.libcxxStdenv; })); + toJSON sets; buildCommand = '' touch $out ''; @@ -104,6 +107,7 @@ with pkgs; cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; }; fetchurl = callPackages ../build-support/fetchurl/tests.nix { }; + fetchtorrent = callPackages ../build-support/fetchtorrent/tests.nix { }; fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { }; fetchpatch2 = callPackages ../build-support/fetchpatch/tests.nix { fetchpatch = fetchpatch2; }; fetchDebianPatch = callPackages ../build-support/fetchdebianpatch/tests.nix { }; diff --git a/pkgs/test/haskell/upstreamStackHpackVersion/default.nix b/pkgs/test/haskell/upstreamStackHpackVersion/default.nix index ddf26770259..f3ddbcd3e01 100644 --- a/pkgs/test/haskell/upstreamStackHpackVersion/default.nix +++ b/pkgs/test/haskell/upstreamStackHpackVersion/default.nix @@ -25,7 +25,7 @@ let # This is a statically linked version of stack, so it should be usable within # the Nixpkgs builder (at least on x86_64-linux). stackDownloadUrl = - "https://github.com/commercialhaskell/stack/releases/download/v${stack.version}/stack-${stack.version}-linux-x86_64-static.tar.gz"; + "https://github.com/commercialhaskell/stack/releases/download/v${stack.version}/stack-${stack.version}-linux-x86_64.tar.gz"; # This test code has been explicitly pulled out of the derivation below so # that it can be hashed and added to the `name` of the derivation. This is @@ -49,10 +49,11 @@ let ) # Fetch the statically-linked upstream Stack binary. + echo "Trying to download a statically linked stack binary from ${stackDownloadUrl} to ./stack.tar.gz ..." "''${curl[@]}" "${stackDownloadUrl}" > ./stack.tar.gz tar xf ./stack.tar.gz - upstream_stack_version_output="$(./stack-${stack.version}-linux-x86_64-static/stack --version)" + upstream_stack_version_output="$(./stack-${stack.version}-linux-x86_64/stack --version)" echo "upstream \`stack --version\` output: $upstream_stack_version_output" nixpkgs_stack_version_output="$(stack --version)" diff --git a/pkgs/test/make-binary-wrapper/add-flags.c b/pkgs/test/make-binary-wrapper/add-flags.c index 3ae8678d442..d998a5f6f98 100644 --- a/pkgs/test/make-binary-wrapper/add-flags.c +++ b/pkgs/test/make-binary-wrapper/add-flags.c @@ -3,19 +3,21 @@ #include <assert.h> int main(int argc, char **argv) { - char **argv_tmp = calloc(4 + argc + 2 + 1, sizeof(*argv_tmp)); + char **argv_tmp = calloc(6 + argc + 2 + 1, sizeof(*argv_tmp)); assert(argv_tmp != NULL); argv_tmp[0] = argv[0]; argv_tmp[1] = "-x"; argv_tmp[2] = "-y"; argv_tmp[3] = "-z"; argv_tmp[4] = "-abc"; + argv_tmp[5] = "-g"; + argv_tmp[6] = "*.txt"; for (int i = 1; i < argc; ++i) { - argv_tmp[4 + i] = argv[i]; + argv_tmp[6 + i] = argv[i]; } - argv_tmp[4 + argc + 0] = "-foo"; - argv_tmp[4 + argc + 1] = "-bar"; - argv_tmp[4 + argc + 2] = NULL; + argv_tmp[6 + argc + 0] = "-foo"; + argv_tmp[6 + argc + 1] = "-bar"; + argv_tmp[6 + argc + 2] = NULL; argv = argv_tmp; argv[0] = "/send/me/flags"; diff --git a/pkgs/test/make-binary-wrapper/add-flags.cmdline b/pkgs/test/make-binary-wrapper/add-flags.cmdline index f42d26f3adf..1ca964ab4e7 100644 --- a/pkgs/test/make-binary-wrapper/add-flags.cmdline +++ b/pkgs/test/make-binary-wrapper/add-flags.cmdline @@ -1,3 +1,4 @@ --append-flags "-foo -bar" \ --add-flags "-x -y -z" \ - --add-flags -abc + --add-flags -abc \ + --add-flags "-g *.txt" diff --git a/pkgs/test/make-binary-wrapper/add-flags.env b/pkgs/test/make-binary-wrapper/add-flags.env index 3626b8cf97b..f0641ef36f7 100644 --- a/pkgs/test/make-binary-wrapper/add-flags.env +++ b/pkgs/test/make-binary-wrapper/add-flags.env @@ -4,5 +4,7 @@ SUBST_ARGV0 -y -z -abc +-g +*.txt -foo -bar diff --git a/pkgs/test/nixpkgs-check-by-name/Cargo.lock b/pkgs/test/nixpkgs-check-by-name/Cargo.lock index aa4459c7cff..fc3aeb9fd79 100644 --- a/pkgs/test/nixpkgs-check-by-name/Cargo.lock +++ b/pkgs/test/nixpkgs-check-by-name/Cargo.lock @@ -163,6 +163,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" [[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] name = "errno" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -219,6 +225,15 @@ dependencies = [ ] [[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] name = "itoa" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -274,6 +289,7 @@ dependencies = [ "anyhow", "clap", "colored", + "itertools", "lazy_static", "regex", "rnix", diff --git a/pkgs/test/nixpkgs-check-by-name/Cargo.toml b/pkgs/test/nixpkgs-check-by-name/Cargo.toml index 70b44d04820..1e6eaa1106d 100644 --- a/pkgs/test/nixpkgs-check-by-name/Cargo.toml +++ b/pkgs/test/nixpkgs-check-by-name/Cargo.toml @@ -13,6 +13,7 @@ serde = { version = "1.0.185", features = ["derive"] } anyhow = "1.0" lazy_static = "1.4.0" colored = "2.0.4" +itertools = "0.11.0" [dev-dependencies] temp-env = "0.3.5" diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 4dd694acd2f..146cea0a64b 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -1,6 +1,6 @@ # Nixpkgs pkgs/by-name checker -This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory once introduced. +This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory. It is being used by [this GitHub Actions workflow](../../../.github/workflows/check-by-name.yml). This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140). @@ -11,13 +11,20 @@ This API may be changed over time if the CI workflow making use of it is adjuste - Command line: `nixpkgs-check-by-name <NIXPKGS>` - Arguments: - `<NIXPKGS>`: The path to the Nixpkgs to check + - `--version <VERSION>`: The version of the checks to perform. + + Possible values: + - `v0` (default) + - `v1` + + See [validation](#validity-checks) for the differences. - Exit code: - `0`: If the [validation](#validity-checks) is successful - `1`: If the [validation](#validity-checks) is not successful - `2`: If an unexpected I/O error occurs - Standard error: - Informative messages - - Error messages if validation is not successful + - Detected problems if validation is not successful ## Validity checks @@ -35,6 +42,7 @@ These checks are performed by this tool: ### Nix evaluation checks - `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`. + - **Only after --version v1**: If `pkgs.${name}` is not auto-called from `pkgs/by-name`, `args` must not be empty - `pkgs.lib.isDerivation pkgs.${name}` is `true`. ## Development diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/pkgs/test/nixpkgs-check-by-name/src/eval.nix index 7c0ae755215..bf9f19d8e46 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.nix +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.nix @@ -18,19 +18,42 @@ let callPackage = fn: args: let result = super.callPackage fn args; + variantInfo._attributeVariant = { + # These names are used by the deserializer on the Rust side + CallPackage.path = + if builtins.isPath fn then + toString fn + else + null; + CallPackage.empty_arg = + args == { }; + }; in if builtins.isAttrs result then # If this was the last overlay to be applied, we could just only return the `_callPackagePath`, # but that's not the case because stdenv has another overlays on top of user-provided ones. # So to not break the stdenv build we need to return the mostly proper result here - result // { - _callPackagePath = fn; - } + result // variantInfo else # It's very rare that callPackage doesn't return an attribute set, but it can occur. - { - _callPackagePath = fn; + variantInfo; + + _internalCallByNamePackageFile = file: + let + result = super._internalCallByNamePackageFile file; + variantInfo._attributeVariant = { + # This name is used by the deserializer on the Rust side + AutoCalled = null; }; + in + if builtins.isAttrs result then + # If this was the last overlay to be applied, we could just only return the `_callPackagePath`, + # but that's not the case because stdenv has another overlays on top of user-provided ones. + # So to not break the stdenv build we need to return the mostly proper result here + result // variantInfo + else + # It's very rare that callPackage doesn't return an attribute set, but it can occur. + variantInfo; }; pkgs = import nixpkgsPath { @@ -39,14 +62,14 @@ let overlays = [ callPackageOverlay ]; }; - attrInfo = attr: { + attrInfo = attr: + let + value = pkgs.${attr}; + in + { # These names are used by the deserializer on the Rust side - call_package_path = - if pkgs.${attr} ? _callPackagePath && builtins.isPath pkgs.${attr}._callPackagePath then - toString pkgs.${attr}._callPackagePath - else - null; - is_derivation = pkgs.lib.isDerivation pkgs.${attr}; + variant = value._attributeVariant or { Other = null; }; + is_derivation = pkgs.lib.isDerivation value; }; attrInfos = builtins.listToAttrs (map (name: { diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index 17e22495b22..e4f986748b4 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -1,11 +1,12 @@ +use crate::nixpkgs_problem::NixpkgsProblem; use crate::structure; -use crate::utils::ErrorWriter; +use crate::validation::{self, Validation::Success}; +use crate::Version; use std::path::Path; use anyhow::Context; use serde::Deserialize; use std::collections::HashMap; -use std::io; use std::path::PathBuf; use std::process; use tempfile::NamedTempFile; @@ -13,20 +14,38 @@ use tempfile::NamedTempFile; /// Attribute set of this structure is returned by eval.nix #[derive(Deserialize)] struct AttributeInfo { - call_package_path: Option<PathBuf>, + variant: AttributeVariant, is_derivation: bool, } +#[derive(Deserialize)] +enum AttributeVariant { + /// The attribute is auto-called as pkgs.callPackage using pkgs/by-name, + /// and it is not overridden by a definition in all-packages.nix + AutoCalled, + /// The attribute is defined as a pkgs.callPackage <path> <args>, + /// and overridden by all-packages.nix + CallPackage { + /// The <path> argument or None if it's not a path + path: Option<PathBuf>, + /// true if <args> is { } + empty_arg: bool, + }, + /// The attribute is not defined as pkgs.callPackage + Other, +} + const EXPR: &str = include_str!("eval.nix"); /// Check that the Nixpkgs attribute values corresponding to the packages in pkgs/by-name are /// of the form `callPackage <package_file> { ... }`. /// See the `eval.nix` file for how this is achieved on the Nix side -pub fn check_values<W: io::Write>( - error_writer: &mut ErrorWriter<W>, - nixpkgs: &structure::Nixpkgs, +pub fn check_values( + version: Version, + nixpkgs_path: &Path, + package_names: Vec<String>, eval_accessible_paths: Vec<&Path>, -) -> anyhow::Result<()> { +) -> validation::Result<()> { // Write the list of packages we need to check into a temporary JSON file. // This can then get read by the Nix evaluation. let attrs_file = NamedTempFile::new().context("Failed to create a temporary file")?; @@ -36,7 +55,7 @@ pub fn check_values<W: io::Write>( // entry is needed. let attrs_file_path = attrs_file.path().canonicalize()?; - serde_json::to_writer(&attrs_file, &nixpkgs.package_names).context(format!( + serde_json::to_writer(&attrs_file, &package_names).context(format!( "Failed to serialise the package names to the temporary path {}", attrs_file_path.display() ))?; @@ -68,9 +87,9 @@ pub fn check_values<W: io::Write>( .arg(&attrs_file_path) // Same for the nixpkgs to test .args(["--arg", "nixpkgsPath"]) - .arg(&nixpkgs.path) + .arg(nixpkgs_path) .arg("-I") - .arg(&nixpkgs.path); + .arg(nixpkgs_path); // Also add extra paths that need to be accessible for path in eval_accessible_paths { @@ -92,39 +111,54 @@ pub fn check_values<W: io::Write>( String::from_utf8_lossy(&result.stdout) ))?; - for package_name in &nixpkgs.package_names { - let relative_package_file = structure::Nixpkgs::relative_file_for_package(package_name); - let absolute_package_file = nixpkgs.path.join(&relative_package_file); + Ok(validation::sequence_(package_names.iter().map( + |package_name| { + let relative_package_file = structure::relative_file_for_package(package_name); + let absolute_package_file = nixpkgs_path.join(&relative_package_file); - if let Some(attribute_info) = actual_files.get(package_name) { - let is_expected_file = - if let Some(call_package_path) = &attribute_info.call_package_path { - absolute_package_file == *call_package_path - } else { - false + if let Some(attribute_info) = actual_files.get(package_name) { + let valid = match &attribute_info.variant { + AttributeVariant::AutoCalled => true, + AttributeVariant::CallPackage { path, empty_arg } => { + let correct_file = if let Some(call_package_path) = path { + absolute_package_file == *call_package_path + } else { + false + }; + // Only check for the argument to be non-empty if the version is V1 or + // higher + let non_empty = if version >= Version::V1 { + !empty_arg + } else { + true + }; + correct_file && non_empty + } + AttributeVariant::Other => false, }; - if !is_expected_file { - error_writer.write(&format!( - "pkgs.{package_name}: This attribute is not defined as `pkgs.callPackage {} {{ ... }}`.", - relative_package_file.display() - ))?; - continue; - } - - if !attribute_info.is_derivation { - error_writer.write(&format!( - "pkgs.{package_name}: This attribute defined by {} is not a derivation", - relative_package_file.display() - ))?; + if !valid { + NixpkgsProblem::WrongCallPackage { + relative_package_file: relative_package_file.clone(), + package_name: package_name.clone(), + } + .into() + } else if !attribute_info.is_derivation { + NixpkgsProblem::NonDerivation { + relative_package_file: relative_package_file.clone(), + package_name: package_name.clone(), + } + .into() + } else { + Success(()) + } + } else { + NixpkgsProblem::UndefinedAttr { + relative_package_file: relative_package_file.clone(), + package_name: package_name.clone(), + } + .into() } - } else { - error_writer.write(&format!( - "pkgs.{package_name}: This attribute is not defined but it should be defined automatically as {}", - relative_package_file.display() - ))?; - continue; - } - } - Ok(()) + }, + ))) } diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 8c663061bb0..4cabf8f446f 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -1,28 +1,45 @@ mod eval; +mod nixpkgs_problem; mod references; mod structure; mod utils; +mod validation; +use crate::structure::check_structure; +use crate::validation::Validation::Failure; +use crate::validation::Validation::Success; use anyhow::Context; -use clap::Parser; +use clap::{Parser, ValueEnum}; use colored::Colorize; use std::io; use std::path::{Path, PathBuf}; use std::process::ExitCode; -use structure::Nixpkgs; -use utils::ErrorWriter; /// Program to check the validity of pkgs/by-name #[derive(Parser, Debug)] #[command(about)] -struct Args { +pub struct Args { /// Path to nixpkgs nixpkgs: PathBuf, + /// The version of the checks + /// Increasing this may cause failures for a Nixpkgs that succeeded before + /// TODO: Remove default once Nixpkgs CI passes this argument + #[arg(long, value_enum, default_value_t = Version::V0)] + version: Version, +} + +/// The version of the checks +#[derive(Debug, Clone, PartialEq, PartialOrd, ValueEnum)] +pub enum Version { + /// Initial version + V0, + /// Empty argument check + V1, } fn main() -> ExitCode { let args = Args::parse(); - match check_nixpkgs(&args.nixpkgs, vec![], &mut io::stderr()) { + match check_nixpkgs(&args.nixpkgs, args.version, vec![], &mut io::stderr()) { Ok(true) => { eprintln!("{}", "Validated successfully".green()); ExitCode::SUCCESS @@ -49,10 +66,11 @@ fn main() -> ExitCode { /// /// # Return value /// - `Err(e)` if an I/O-related error `e` occurred. -/// - `Ok(false)` if the structure is invalid, all the structural errors have been written to `error_writer`. -/// - `Ok(true)` if the structure is valid, nothing will have been written to `error_writer`. +/// - `Ok(false)` if there are problems, all of which will be written to `error_writer`. +/// - `Ok(true)` if there are no problems pub fn check_nixpkgs<W: io::Write>( nixpkgs_path: &Path, + version: Version, eval_accessible_paths: Vec<&Path>, error_writer: &mut W, ) -> anyhow::Result<bool> { @@ -61,31 +79,39 @@ pub fn check_nixpkgs<W: io::Write>( nixpkgs_path.display() ))?; - // Wraps the error_writer to print everything in red, and tracks whether anything was printed - // at all. Later used to figure out if the structure was valid or not. - let mut error_writer = ErrorWriter::new(error_writer); - - if !nixpkgs_path.join(structure::BASE_SUBPATH).exists() { + let check_result = if !nixpkgs_path.join(utils::BASE_SUBPATH).exists() { eprintln!( "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.", - structure::BASE_SUBPATH + utils::BASE_SUBPATH ); + Success(()) } else { - let nixpkgs = Nixpkgs::new(&nixpkgs_path, &mut error_writer)?; + match check_structure(&nixpkgs_path)? { + Failure(errors) => Failure(errors), + Success(package_names) => + // Only if we could successfully parse the structure, we do the evaluation checks + { + eval::check_values(version, &nixpkgs_path, package_names, eval_accessible_paths)? + } + } + }; - if error_writer.empty { - // Only if we could successfully parse the structure, we do the semantic checks - eval::check_values(&mut error_writer, &nixpkgs, eval_accessible_paths)?; - references::check_references(&mut error_writer, &nixpkgs)?; + match check_result { + Failure(errors) => { + for error in errors { + writeln!(error_writer, "{}", error.to_string().red())? + } + Ok(false) } + Success(_) => Ok(true), } - Ok(error_writer.empty) } #[cfg(test)] mod tests { use crate::check_nixpkgs; - use crate::structure; + use crate::utils; + use crate::Version; use anyhow::Context; use std::fs; use std::path::Path; @@ -129,7 +155,7 @@ mod tests { return Ok(()); } - let base = path.join(structure::BASE_SUBPATH); + let base = path.join(utils::BASE_SUBPATH); fs::create_dir_all(base.join("fo/foo"))?; fs::write(base.join("fo/foo/package.nix"), "{ someDrv }: someDrv")?; @@ -174,7 +200,7 @@ mod tests { // We don't want coloring to mess up the tests let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> { let mut writer = vec![]; - check_nixpkgs(&path, vec![&extra_nix_path], &mut writer) + check_nixpkgs(&path, Version::V1, vec![&extra_nix_path], &mut writer) .context(format!("Failed test case {name}"))?; Ok(writer) })?; diff --git a/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs b/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs new file mode 100644 index 00000000000..2344a8cc132 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs @@ -0,0 +1,218 @@ +use crate::utils::PACKAGE_NIX_FILENAME; +use rnix::parser::ParseError; +use std::ffi::OsString; +use std::fmt; +use std::io; +use std::path::PathBuf; + +/// Any problem that can occur when checking Nixpkgs +pub enum NixpkgsProblem { + ShardNonDir { + relative_shard_path: PathBuf, + }, + InvalidShardName { + relative_shard_path: PathBuf, + shard_name: String, + }, + PackageNonDir { + relative_package_dir: PathBuf, + }, + CaseSensitiveDuplicate { + relative_shard_path: PathBuf, + first: OsString, + second: OsString, + }, + InvalidPackageName { + relative_package_dir: PathBuf, + package_name: String, + }, + IncorrectShard { + relative_package_dir: PathBuf, + correct_relative_package_dir: PathBuf, + }, + PackageNixNonExistent { + relative_package_dir: PathBuf, + }, + PackageNixDir { + relative_package_dir: PathBuf, + }, + UndefinedAttr { + relative_package_file: PathBuf, + package_name: String, + }, + WrongCallPackage { + relative_package_file: PathBuf, + package_name: String, + }, + NonDerivation { + relative_package_file: PathBuf, + package_name: String, + }, + OutsideSymlink { + relative_package_dir: PathBuf, + subpath: PathBuf, + }, + UnresolvableSymlink { + relative_package_dir: PathBuf, + subpath: PathBuf, + io_error: io::Error, + }, + CouldNotParseNix { + relative_package_dir: PathBuf, + subpath: PathBuf, + error: ParseError, + }, + PathInterpolation { + relative_package_dir: PathBuf, + subpath: PathBuf, + line: usize, + text: String, + }, + SearchPath { + relative_package_dir: PathBuf, + subpath: PathBuf, + line: usize, + text: String, + }, + OutsidePathReference { + relative_package_dir: PathBuf, + subpath: PathBuf, + line: usize, + text: String, + }, + UnresolvablePathReference { + relative_package_dir: PathBuf, + subpath: PathBuf, + line: usize, + text: String, + io_error: io::Error, + }, +} + +impl fmt::Display for NixpkgsProblem { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + NixpkgsProblem::ShardNonDir { relative_shard_path } => + write!( + f, + "{}: This is a file, but it should be a directory.", + relative_shard_path.display(), + ), + NixpkgsProblem::InvalidShardName { relative_shard_path, shard_name } => + write!( + f, + "{}: Invalid directory name \"{shard_name}\", must be at most 2 ASCII characters consisting of a-z, 0-9, \"-\" or \"_\".", + relative_shard_path.display() + ), + NixpkgsProblem::PackageNonDir { relative_package_dir } => + write!( + f, + "{}: This path is a file, but it should be a directory.", + relative_package_dir.display(), + ), + NixpkgsProblem::CaseSensitiveDuplicate { relative_shard_path, first, second } => + write!( + f, + "{}: Duplicate case-sensitive package directories {first:?} and {second:?}.", + relative_shard_path.display(), + ), + NixpkgsProblem::InvalidPackageName { relative_package_dir, package_name } => + write!( + f, + "{}: Invalid package directory name \"{package_name}\", must be ASCII characters consisting of a-z, A-Z, 0-9, \"-\" or \"_\".", + relative_package_dir.display(), + ), + NixpkgsProblem::IncorrectShard { relative_package_dir, correct_relative_package_dir } => + write!( + f, + "{}: Incorrect directory location, should be {} instead.", + relative_package_dir.display(), + correct_relative_package_dir.display(), + ), + NixpkgsProblem::PackageNixNonExistent { relative_package_dir } => + write!( + f, + "{}: Missing required \"{PACKAGE_NIX_FILENAME}\" file.", + relative_package_dir.display(), + ), + NixpkgsProblem::PackageNixDir { relative_package_dir } => + write!( + f, + "{}: \"{PACKAGE_NIX_FILENAME}\" must be a file.", + relative_package_dir.display(), + ), + NixpkgsProblem::UndefinedAttr { relative_package_file, package_name } => + write!( + f, + "pkgs.{package_name}: This attribute is not defined but it should be defined automatically as {}", + relative_package_file.display() + ), + NixpkgsProblem::WrongCallPackage { relative_package_file, package_name } => + write!( + f, + "pkgs.{package_name}: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage {} {{ ... }}` with a non-empty second argument.", + relative_package_file.display() + ), + NixpkgsProblem::NonDerivation { relative_package_file, package_name } => + write!( + f, + "pkgs.{package_name}: This attribute defined by {} is not a derivation", + relative_package_file.display() + ), + NixpkgsProblem::OutsideSymlink { relative_package_dir, subpath } => + write!( + f, + "{}: Path {} is a symlink pointing to a path outside the directory of that package.", + relative_package_dir.display(), + subpath.display(), + ), + NixpkgsProblem::UnresolvableSymlink { relative_package_dir, subpath, io_error } => + write!( + f, + "{}: Path {} is a symlink which cannot be resolved: {io_error}.", + relative_package_dir.display(), + subpath.display(), + ), + NixpkgsProblem::CouldNotParseNix { relative_package_dir, subpath, error } => + write!( + f, + "{}: File {} could not be parsed by rnix: {}", + relative_package_dir.display(), + subpath.display(), + error, + ), + NixpkgsProblem::PathInterpolation { relative_package_dir, subpath, line, text } => + write!( + f, + "{}: File {} at line {line} contains the path expression \"{}\", which is not yet supported and may point outside the directory of that package.", + relative_package_dir.display(), + subpath.display(), + text + ), + NixpkgsProblem::SearchPath { relative_package_dir, subpath, line, text } => + write!( + f, + "{}: File {} at line {line} contains the nix search path expression \"{}\" which may point outside the directory of that package.", + relative_package_dir.display(), + subpath.display(), + text + ), + NixpkgsProblem::OutsidePathReference { relative_package_dir, subpath, line, text } => + write!( + f, + "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.", + relative_package_dir.display(), + subpath.display(), + text, + ), + NixpkgsProblem::UnresolvablePathReference { relative_package_dir, subpath, line, text, io_error } => + write!( + f, + "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {io_error}.", + relative_package_dir.display(), + subpath.display(), + text, + ), + } + } +} diff --git a/pkgs/test/nixpkgs-check-by-name/src/references.rs b/pkgs/test/nixpkgs-check-by-name/src/references.rs index 16dc60729c4..0561a9b22e8 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/references.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/references.rs @@ -1,105 +1,98 @@ -use crate::structure::Nixpkgs; +use crate::nixpkgs_problem::NixpkgsProblem; use crate::utils; -use crate::utils::{ErrorWriter, LineIndex}; +use crate::utils::LineIndex; +use crate::validation::{self, ResultIteratorExt, Validation::Success}; use anyhow::Context; use rnix::{Root, SyntaxKind::NODE_PATH}; use std::ffi::OsStr; use std::fs::read_to_string; -use std::io; -use std::path::{Path, PathBuf}; - -/// Small helper so we don't need to pass in the same arguments to all functions -struct PackageContext<'a, W: io::Write> { - error_writer: &'a mut ErrorWriter<W>, - /// The package directory relative to Nixpkgs, such as `pkgs/by-name/fo/foo` - relative_package_dir: &'a PathBuf, - /// The absolute package directory - absolute_package_dir: &'a PathBuf, -} +use std::path::Path; /// Check that every package directory in pkgs/by-name doesn't link to outside that directory. /// Both symlinks and Nix path expressions are checked. -pub fn check_references<W: io::Write>( - error_writer: &mut ErrorWriter<W>, - nixpkgs: &Nixpkgs, -) -> anyhow::Result<()> { - // Check the directories for each package separately - for package_name in &nixpkgs.package_names { - let relative_package_dir = Nixpkgs::relative_dir_for_package(package_name); - let mut context = PackageContext { - error_writer, - relative_package_dir: &relative_package_dir, - absolute_package_dir: &nixpkgs.path.join(&relative_package_dir), - }; - - // The empty argument here is the subpath under the package directory to check - // An empty one means the package directory itself - check_path(&mut context, Path::new("")).context(format!( - "While checking the references in package directory {}", - relative_package_dir.display() - ))?; - } - Ok(()) +pub fn check_references( + relative_package_dir: &Path, + absolute_package_dir: &Path, +) -> validation::Result<()> { + // The empty argument here is the subpath under the package directory to check + // An empty one means the package directory itself + check_path(relative_package_dir, absolute_package_dir, Path::new("")).context(format!( + "While checking the references in package directory {}", + relative_package_dir.display() + )) } /// Checks for a specific path to not have references outside -fn check_path<W: io::Write>(context: &mut PackageContext<W>, subpath: &Path) -> anyhow::Result<()> { - let path = context.absolute_package_dir.join(subpath); +fn check_path( + relative_package_dir: &Path, + absolute_package_dir: &Path, + subpath: &Path, +) -> validation::Result<()> { + let path = absolute_package_dir.join(subpath); - if path.is_symlink() { + Ok(if path.is_symlink() { // Check whether the symlink resolves to outside the package directory match path.canonicalize() { Ok(target) => { // No need to handle the case of it being inside the directory, since we scan through the // entire directory recursively anyways - if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) { - context.error_writer.write(&format!( - "{}: Path {} is a symlink pointing to a path outside the directory of that package.", - context.relative_package_dir.display(), - subpath.display(), - ))?; + if let Err(_prefix_error) = target.strip_prefix(absolute_package_dir) { + NixpkgsProblem::OutsideSymlink { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + } + .into() + } else { + Success(()) } } - Err(e) => { - context.error_writer.write(&format!( - "{}: Path {} is a symlink which cannot be resolved: {e}.", - context.relative_package_dir.display(), - subpath.display(), - ))?; + Err(io_error) => NixpkgsProblem::UnresolvableSymlink { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + io_error, } + .into(), } } else if path.is_dir() { // Recursively check each entry - for entry in utils::read_dir_sorted(&path)? { - let entry_subpath = subpath.join(entry.file_name()); - check_path(context, &entry_subpath) - .context(format!("Error while recursing into {}", subpath.display()))? - } + validation::sequence_( + utils::read_dir_sorted(&path)? + .into_iter() + .map(|entry| { + let entry_subpath = subpath.join(entry.file_name()); + check_path(relative_package_dir, absolute_package_dir, &entry_subpath) + .context(format!("Error while recursing into {}", subpath.display())) + }) + .collect_vec()?, + ) } else if path.is_file() { // Only check Nix files if let Some(ext) = path.extension() { if ext == OsStr::new("nix") { - check_nix_file(context, subpath).context(format!( - "Error while checking Nix file {}", - subpath.display() - ))? + check_nix_file(relative_package_dir, absolute_package_dir, subpath).context( + format!("Error while checking Nix file {}", subpath.display()), + )? + } else { + Success(()) } + } else { + Success(()) } } else { // This should never happen, git doesn't support other file types anyhow::bail!("Unsupported file type for path {}", subpath.display()); - } - Ok(()) + }) } /// Check whether a nix file contains path expression references pointing outside the package /// directory -fn check_nix_file<W: io::Write>( - context: &mut PackageContext<W>, +fn check_nix_file( + relative_package_dir: &Path, + absolute_package_dir: &Path, subpath: &Path, -) -> anyhow::Result<()> { - let path = context.absolute_package_dir.join(subpath); +) -> validation::Result<()> { + let path = absolute_package_dir.join(subpath); let parent_dir = path.parent().context(format!( "Could not get parent of path {}", subpath.display() @@ -110,75 +103,73 @@ fn check_nix_file<W: io::Write>( let root = Root::parse(&contents); if let Some(error) = root.errors().first() { - context.error_writer.write(&format!( - "{}: File {} could not be parsed by rnix: {}", - context.relative_package_dir.display(), - subpath.display(), - error, - ))?; - return Ok(()); + return Ok(NixpkgsProblem::CouldNotParseNix { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + error: error.clone(), + } + .into()); } let line_index = LineIndex::new(&contents); - for node in root.syntax().descendants() { - // We're only interested in Path expressions - if node.kind() != NODE_PATH { - continue; - } - - let text = node.text().to_string(); - let line = line_index.line(node.text_range().start().into()); - - // Filters out ./foo/${bar}/baz - // TODO: We can just check ./foo - if node.children().count() != 0 { - context.error_writer.write(&format!( - "{}: File {} at line {line} contains the path expression \"{}\", which is not yet supported and may point outside the directory of that package.", - context.relative_package_dir.display(), - subpath.display(), - text - ))?; - continue; - } - - // Filters out search paths like <nixpkgs> - if text.starts_with('<') { - context.error_writer.write(&format!( - "{}: File {} at line {line} contains the nix search path expression \"{}\" which may point outside the directory of that package.", - context.relative_package_dir.display(), - subpath.display(), - text - ))?; - continue; - } - - // Resolves the reference of the Nix path - // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz` - match parent_dir.join(Path::new(&text)).canonicalize() { - Ok(target) => { - // Then checking if it's still in the package directory - // No need to handle the case of it being inside the directory, since we scan through the - // entire directory recursively anyways - if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) { - context.error_writer.write(&format!( - "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.", - context.relative_package_dir.display(), - subpath.display(), - text, - ))?; + Ok(validation::sequence_(root.syntax().descendants().map( + |node| { + let text = node.text().to_string(); + let line = line_index.line(node.text_range().start().into()); + + if node.kind() != NODE_PATH { + // We're only interested in Path expressions + Success(()) + } else if node.children().count() != 0 { + // Filters out ./foo/${bar}/baz + // TODO: We can just check ./foo + NixpkgsProblem::PathInterpolation { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + line, + text, } - } - Err(e) => { - context.error_writer.write(&format!( - "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {e}.", - context.relative_package_dir.display(), - subpath.display(), + .into() + } else if text.starts_with('<') { + // Filters out search paths like <nixpkgs> + NixpkgsProblem::SearchPath { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + line, text, - ))?; + } + .into() + } else { + // Resolves the reference of the Nix path + // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz` + match parent_dir.join(Path::new(&text)).canonicalize() { + Ok(target) => { + // Then checking if it's still in the package directory + // No need to handle the case of it being inside the directory, since we scan through the + // entire directory recursively anyways + if let Err(_prefix_error) = target.strip_prefix(absolute_package_dir) { + NixpkgsProblem::OutsidePathReference { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + line, + text, + } + .into() + } else { + Success(()) + } + } + Err(e) => NixpkgsProblem::UnresolvablePathReference { + relative_package_dir: relative_package_dir.to_path_buf(), + subpath: subpath.to_path_buf(), + line, + text, + io_error: e, + } + .into(), + } } - }; - } - - Ok(()) + }, + ))) } diff --git a/pkgs/test/nixpkgs-check-by-name/src/structure.rs b/pkgs/test/nixpkgs-check-by-name/src/structure.rs index ea80128e487..4051ca037c9 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/structure.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/structure.rs @@ -1,152 +1,170 @@ +use crate::nixpkgs_problem::NixpkgsProblem; +use crate::references; use crate::utils; -use crate::utils::ErrorWriter; +use crate::utils::{BASE_SUBPATH, PACKAGE_NIX_FILENAME}; +use crate::validation::{self, ResultIteratorExt, Validation::Success}; +use itertools::concat; use lazy_static::lazy_static; use regex::Regex; -use std::collections::HashMap; -use std::io; +use std::fs::DirEntry; use std::path::{Path, PathBuf}; -pub const BASE_SUBPATH: &str = "pkgs/by-name"; -pub const PACKAGE_NIX_FILENAME: &str = "package.nix"; - lazy_static! { static ref SHARD_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_-]{1,2}$").unwrap(); static ref PACKAGE_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_-]+$").unwrap(); } -/// Contains information about the structure of the pkgs/by-name directory of a Nixpkgs -pub struct Nixpkgs { - /// The path to nixpkgs - pub path: PathBuf, - /// The names of all packages declared in pkgs/by-name - pub package_names: Vec<String>, -} - -impl Nixpkgs { - // Some utility functions for the basic structure +// Some utility functions for the basic structure - pub fn shard_for_package(package_name: &str) -> String { - package_name.to_lowercase().chars().take(2).collect() - } - - pub fn relative_dir_for_shard(shard_name: &str) -> PathBuf { - PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}")) - } +pub fn shard_for_package(package_name: &str) -> String { + package_name.to_lowercase().chars().take(2).collect() +} - pub fn relative_dir_for_package(package_name: &str) -> PathBuf { - Nixpkgs::relative_dir_for_shard(&Nixpkgs::shard_for_package(package_name)) - .join(package_name) - } +pub fn relative_dir_for_shard(shard_name: &str) -> PathBuf { + PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}")) +} - pub fn relative_file_for_package(package_name: &str) -> PathBuf { - Nixpkgs::relative_dir_for_package(package_name).join(PACKAGE_NIX_FILENAME) - } +pub fn relative_dir_for_package(package_name: &str) -> PathBuf { + relative_dir_for_shard(&shard_for_package(package_name)).join(package_name) } -impl Nixpkgs { - /// Read the structure of a Nixpkgs directory, displaying errors on the writer. - /// May return early with I/O errors. - pub fn new<W: io::Write>( - path: &Path, - error_writer: &mut ErrorWriter<W>, - ) -> anyhow::Result<Nixpkgs> { - let base_dir = path.join(BASE_SUBPATH); +pub fn relative_file_for_package(package_name: &str) -> PathBuf { + relative_dir_for_package(package_name).join(PACKAGE_NIX_FILENAME) +} - let mut package_names = Vec::new(); +/// Check the structure of Nixpkgs, returning the attribute names that are defined in +/// `pkgs/by-name` +pub fn check_structure(path: &Path) -> validation::Result<Vec<String>> { + let base_dir = path.join(BASE_SUBPATH); - for shard_entry in utils::read_dir_sorted(&base_dir)? { + let shard_results = utils::read_dir_sorted(&base_dir)? + .into_iter() + .map(|shard_entry| -> validation::Result<_> { let shard_path = shard_entry.path(); let shard_name = shard_entry.file_name().to_string_lossy().into_owned(); - let relative_shard_path = Nixpkgs::relative_dir_for_shard(&shard_name); + let relative_shard_path = relative_dir_for_shard(&shard_name); - if shard_name == "README.md" { + Ok(if shard_name == "README.md" { // README.md is allowed to be a file and not checked - continue; - } - - if !shard_path.is_dir() { - error_writer.write(&format!( - "{}: This is a file, but it should be a directory.", - relative_shard_path.display(), - ))?; - // we can't check for any other errors if it's a file, since there's no subdirectories to check - continue; - } - - let shard_name_valid = SHARD_NAME_REGEX.is_match(&shard_name); - if !shard_name_valid { - error_writer.write(&format!( - "{}: Invalid directory name \"{shard_name}\", must be at most 2 ASCII characters consisting of a-z, 0-9, \"-\" or \"_\".", - relative_shard_path.display() - ))?; - } - - let mut unique_package_names = HashMap::new(); - - for package_entry in utils::read_dir_sorted(&shard_path)? { - let package_path = package_entry.path(); - let package_name = package_entry.file_name().to_string_lossy().into_owned(); - let relative_package_dir = - PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}/{package_name}")); - - if !package_path.is_dir() { - error_writer.write(&format!( - "{}: This path is a file, but it should be a directory.", - relative_package_dir.display(), - ))?; - continue; - } - if let Some(duplicate_package_name) = - unique_package_names.insert(package_name.to_lowercase(), package_name.clone()) - { - error_writer.write(&format!( - "{}: Duplicate case-sensitive package directories \"{duplicate_package_name}\" and \"{package_name}\".", - relative_shard_path.display(), - ))?; + Success(vec![]) + } else if !shard_path.is_dir() { + NixpkgsProblem::ShardNonDir { + relative_shard_path: relative_shard_path.clone(), } - - let package_name_valid = PACKAGE_NAME_REGEX.is_match(&package_name); - if !package_name_valid { - error_writer.write(&format!( - "{}: Invalid package directory name \"{package_name}\", must be ASCII characters consisting of a-z, A-Z, 0-9, \"-\" or \"_\".", - relative_package_dir.display(), - ))?; - } - - let correct_relative_package_dir = Nixpkgs::relative_dir_for_package(&package_name); - if relative_package_dir != correct_relative_package_dir { - // Only show this error if we have a valid shard and package name - // Because if one of those is wrong, you should fix that first - if shard_name_valid && package_name_valid { - error_writer.write(&format!( - "{}: Incorrect directory location, should be {} instead.", - relative_package_dir.display(), - correct_relative_package_dir.display(), - ))?; + .into() + // we can't check for any other errors if it's a file, since there's no subdirectories to check + } else { + let shard_name_valid = SHARD_NAME_REGEX.is_match(&shard_name); + let result = if !shard_name_valid { + NixpkgsProblem::InvalidShardName { + relative_shard_path: relative_shard_path.clone(), + shard_name: shard_name.clone(), } - } + .into() + } else { + Success(()) + }; + + let entries = utils::read_dir_sorted(&shard_path)?; + + let duplicate_results = entries + .iter() + .zip(entries.iter().skip(1)) + .filter(|(l, r)| { + l.file_name().to_ascii_lowercase() == r.file_name().to_ascii_lowercase() + }) + .map(|(l, r)| { + NixpkgsProblem::CaseSensitiveDuplicate { + relative_shard_path: relative_shard_path.clone(), + first: l.file_name(), + second: r.file_name(), + } + .into() + }); + + let result = result.and(validation::sequence_(duplicate_results)); + + let package_results = entries + .into_iter() + .map(|package_entry| { + check_package(path, &shard_name, shard_name_valid, package_entry) + }) + .collect_vec()?; + + result.and(validation::sequence(package_results)) + }) + }) + .collect_vec()?; - let package_nix_path = package_path.join(PACKAGE_NIX_FILENAME); - if !package_nix_path.exists() { - error_writer.write(&format!( - "{}: Missing required \"{PACKAGE_NIX_FILENAME}\" file.", - relative_package_dir.display(), - ))?; - } else if package_nix_path.is_dir() { - error_writer.write(&format!( - "{}: \"{PACKAGE_NIX_FILENAME}\" must be a file.", - relative_package_dir.display(), - ))?; - } + // Combine the package names conatained within each shard into a longer list + Ok(validation::sequence(shard_results).map(concat)) +} - package_names.push(package_name.clone()); - } +fn check_package( + path: &Path, + shard_name: &str, + shard_name_valid: bool, + package_entry: DirEntry, +) -> validation::Result<String> { + let package_path = package_entry.path(); + let package_name = package_entry.file_name().to_string_lossy().into_owned(); + let relative_package_dir = PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}/{package_name}")); + + Ok(if !package_path.is_dir() { + NixpkgsProblem::PackageNonDir { + relative_package_dir: relative_package_dir.clone(), } - - Ok(Nixpkgs { - path: path.to_owned(), - package_names, - }) - } + .into() + } else { + let package_name_valid = PACKAGE_NAME_REGEX.is_match(&package_name); + let result = if !package_name_valid { + NixpkgsProblem::InvalidPackageName { + relative_package_dir: relative_package_dir.clone(), + package_name: package_name.clone(), + } + .into() + } else { + Success(()) + }; + + let correct_relative_package_dir = relative_dir_for_package(&package_name); + let result = result.and(if relative_package_dir != correct_relative_package_dir { + // Only show this error if we have a valid shard and package name + // Because if one of those is wrong, you should fix that first + if shard_name_valid && package_name_valid { + NixpkgsProblem::IncorrectShard { + relative_package_dir: relative_package_dir.clone(), + correct_relative_package_dir: correct_relative_package_dir.clone(), + } + .into() + } else { + Success(()) + } + } else { + Success(()) + }); + + let package_nix_path = package_path.join(PACKAGE_NIX_FILENAME); + let result = result.and(if !package_nix_path.exists() { + NixpkgsProblem::PackageNixNonExistent { + relative_package_dir: relative_package_dir.clone(), + } + .into() + } else if package_nix_path.is_dir() { + NixpkgsProblem::PackageNixDir { + relative_package_dir: relative_package_dir.clone(), + } + .into() + } else { + Success(()) + }); + + let result = result.and(references::check_references( + &relative_package_dir, + &path.join(&relative_package_dir), + )?); + + result.map(|_| package_name.clone()) + }) } diff --git a/pkgs/test/nixpkgs-check-by-name/src/utils.rs b/pkgs/test/nixpkgs-check-by-name/src/utils.rs index 325c736eca9..5cc4a0863ba 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/utils.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/utils.rs @@ -1,9 +1,11 @@ use anyhow::Context; -use colored::Colorize; use std::fs; use std::io; use std::path::Path; +pub const BASE_SUBPATH: &str = "pkgs/by-name"; +pub const PACKAGE_NIX_FILENAME: &str = "package.nix"; + /// Deterministic file listing so that tests are reproducible pub fn read_dir_sorted(base_dir: &Path) -> anyhow::Result<Vec<fs::DirEntry>> { let listing = base_dir @@ -47,26 +49,3 @@ impl LineIndex { } } } - -/// A small wrapper around a generic io::Write specifically for errors: -/// - Print everything in red to signal it's an error -/// - Keep track of whether anything was printed at all, so that -/// it can be queried whether any errors were encountered at all -pub struct ErrorWriter<W> { - pub writer: W, - pub empty: bool, -} - -impl<W: io::Write> ErrorWriter<W> { - pub fn new(writer: W) -> ErrorWriter<W> { - ErrorWriter { - writer, - empty: true, - } - } - - pub fn write(&mut self, string: &str) -> io::Result<()> { - self.empty = false; - writeln!(self.writer, "{}", string.red()) - } -} diff --git a/pkgs/test/nixpkgs-check-by-name/src/validation.rs b/pkgs/test/nixpkgs-check-by-name/src/validation.rs new file mode 100644 index 00000000000..e7279385152 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/src/validation.rs @@ -0,0 +1,102 @@ +use crate::nixpkgs_problem::NixpkgsProblem; +use itertools::concat; +use itertools::{ + Either::{Left, Right}, + Itertools, +}; +use Validation::*; + +/// The validation result of a check. +/// Instead of exiting at the first failure, +/// this type can accumulate multiple failures. +/// This can be achieved using the functions `and`, `sequence` and `sequence_` +/// +/// This leans on https://hackage.haskell.org/package/validation +pub enum Validation<A> { + Failure(Vec<NixpkgsProblem>), + Success(A), +} + +impl<A> From<NixpkgsProblem> for Validation<A> { + /// Create a `Validation<A>` from a single check problem + fn from(value: NixpkgsProblem) -> Self { + Failure(vec![value]) + } +} + +/// A type alias representing the result of a check, either: +/// - Err(anyhow::Error): A fatal failure, typically I/O errors. +/// Such failures are not caused by the files in Nixpkgs. +/// This hints at a bug in the code or a problem with the deployment. +/// - Ok(Failure(Vec<NixpkgsProblem>)): A non-fatal validation problem with the Nixpkgs files. +/// Further checks can be run even with this result type. +/// Such problems can be fixed by changing the Nixpkgs files. +/// - Ok(Success(A)): A successful (potentially intermediate) result with an arbitrary value. +/// No fatal errors have occurred and no validation problems have been found with Nixpkgs. +pub type Result<A> = anyhow::Result<Validation<A>>; + +pub trait ResultIteratorExt<A, E>: Sized + Iterator<Item = std::result::Result<A, E>> { + fn collect_vec(self) -> std::result::Result<Vec<A>, E>; +} + +impl<I, A, E> ResultIteratorExt<A, E> for I +where + I: Sized + Iterator<Item = std::result::Result<A, E>>, +{ + /// A convenience version of `collect` specialised to a vector + fn collect_vec(self) -> std::result::Result<Vec<A>, E> { + self.collect() + } +} + +impl<A> Validation<A> { + /// Map a `Validation<A>` to a `Validation<B>` by applying a function to the + /// potentially contained value in case of success. + pub fn map<B>(self, f: impl FnOnce(A) -> B) -> Validation<B> { + match self { + Failure(err) => Failure(err), + Success(value) => Success(f(value)), + } + } +} + +impl Validation<()> { + /// Combine two validations, both of which need to be successful for the return value to be successful. + /// The `NixpkgsProblem`s of both sides are returned concatenated. + pub fn and<A>(self, other: Validation<A>) -> Validation<A> { + match (self, other) { + (Success(_), Success(right_value)) => Success(right_value), + (Failure(errors), Success(_)) => Failure(errors), + (Success(_), Failure(errors)) => Failure(errors), + (Failure(errors_l), Failure(errors_r)) => Failure(concat([errors_l, errors_r])), + } + } +} + +/// Combine many validations into a single one. +/// All given validations need to be successful in order for the returned validation to be successful, +/// in which case the returned validation value contains a `Vec` of each individual value. +/// Otherwise the `NixpkgsProblem`s of all validations are returned concatenated. +pub fn sequence<A>(check_results: impl IntoIterator<Item = Validation<A>>) -> Validation<Vec<A>> { + let (errors, values): (Vec<Vec<NixpkgsProblem>>, Vec<A>) = check_results + .into_iter() + .partition_map(|validation| match validation { + Failure(err) => Left(err), + Success(value) => Right(value), + }); + + // To combine the errors from the results we flatten all the error Vec's into a new Vec + // This is not very efficient, but doesn't matter because generally we should have no errors + let flattened_errors = errors.into_iter().concat(); + + if flattened_errors.is_empty() { + Success(values) + } else { + Failure(flattened_errors) + } +} + +/// Like `sequence`, but without any containing value, for convenience +pub fn sequence_(validations: impl IntoIterator<Item = Validation<()>>) -> Validation<()> { + sequence(validations).map(|_| ()) +} diff --git a/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix index 50ad6761754..01bb27a4803 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix +++ b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix @@ -75,9 +75,14 @@ let # Turns autoCalledPackageFiles into an overlay that `callPackage`'s all of them autoCalledPackages = self: super: - builtins.mapAttrs (name: file: - self.callPackage file { } - ) autoCalledPackageFiles; + { + # Needed to be able to detect empty arguments in all-packages.nix + # See a more detailed description in pkgs/top-level/by-name-overlay.nix + _internalCallByNamePackageFile = file: self.callPackage file { }; + } + // builtins.mapAttrs + (name: self._internalCallByNamePackageFile) + autoCalledPackageFiles; # A list optionally containing the `all-packages.nix` file from the test case as an overlay optionalAllPackagesOverlay = diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected index 1c6377d8aef..51479e22d26 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected @@ -1 +1 @@ -pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`. +pkgs.nonDerivation: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }` with a non-empty second argument. diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/all-packages.nix new file mode 100644 index 00000000000..d369dd7228d --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/all-packages.nix @@ -0,0 +1,3 @@ +self: super: { + nonDerivation = self.callPackage ./pkgs/by-name/no/nonDerivation/package.nix { }; +} diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix new file mode 100644 index 00000000000..af25d145012 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix @@ -0,0 +1 @@ +import ../mock-nixpkgs.nix { root = ./.; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/expected new file mode 100644 index 00000000000..51479e22d26 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/expected @@ -0,0 +1 @@ +pkgs.nonDerivation: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }` with a non-empty second argument. diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/pkgs/by-name/no/nonDerivation/package.nix new file mode 100644 index 00000000000..a1b92efbbad --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/pkgs/by-name/no/nonDerivation/package.nix @@ -0,0 +1 @@ +{ someDrv }: someDrv diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected index 1c6377d8aef..51479e22d26 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected @@ -1 +1 @@ -pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`. +pkgs.nonDerivation: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }` with a non-empty second argument. diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected index 1c6377d8aef..51479e22d26 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected @@ -1 +1 @@ -pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`. +pkgs.nonDerivation: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }` with a non-empty second argument. diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-success/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-success/all-packages.nix new file mode 100644 index 00000000000..6b323756ae4 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-success/all-packages.nix @@ -0,0 +1,5 @@ +self: super: { + foo = self.callPackage ./pkgs/by-name/fo/foo/package.nix { + enableBar = true; + }; +} diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix new file mode 100644 index 00000000000..af25d145012 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix @@ -0,0 +1 @@ +import ../mock-nixpkgs.nix { root = ./.; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-success/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-success/pkgs/by-name/fo/foo/package.nix new file mode 100644 index 00000000000..c811a7215a2 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-success/pkgs/by-name/fo/foo/package.nix @@ -0,0 +1,8 @@ +{ + someDrv, + enableBar ? false, +}: +if enableBar then + someDrv +else + {} diff --git a/pkgs/test/simple/builder.sh b/pkgs/test/simple/builder.sh index 908faec3c38..0b09a109bea 100644 --- a/pkgs/test/simple/builder.sh +++ b/pkgs/test/simple/builder.sh @@ -1,4 +1,4 @@ -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi set -x export NIX_DEBUG=1 diff --git a/pkgs/test/stdenv/default.nix b/pkgs/test/stdenv/default.nix index 0fa87cccc21..3882eb2b625 100644 --- a/pkgs/test/stdenv/default.nix +++ b/pkgs/test/stdenv/default.nix @@ -142,6 +142,15 @@ in ''; }; + # Check that mkDerivation rejects MD5 hashes + rejectedHashes = lib.recurseIntoAttrs { + md5 = + let drv = runCommand "md5 outputHash rejected" { + outputHash = "md5-fPt7dxVVP7ffY3MxkQdwVw=="; + } "true"; + in assert !(builtins.tryEval drv).success; {}; + }; + test-inputDerivation = let inherit (stdenv.mkDerivation { dep1 = derivation { name = "dep1"; builder = "/bin/sh"; args = [ "-c" ": > $out" ]; system = builtins.currentSystem; }; diff --git a/pkgs/test/texlive/default.nix b/pkgs/test/texlive/default.nix index 12c42444f18..12fdd5c45f8 100644 --- a/pkgs/test/texlive/default.nix +++ b/pkgs/test/texlive/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText }: +{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText, texliveInfraOnly, texliveSmall, texliveMedium, texliveFull }: rec { @@ -6,7 +6,7 @@ rec { { name , format , text - , texLive ? texlive.combined.scheme-small + , texLive ? texliveSmall , options ? "-interaction=errorstopmode" , preTest ? "" , postTest ? "" @@ -43,7 +43,7 @@ rec { lualatex = mkTeXTest { name = "opentype-fonts-lualatex"; format = "lualatex"; - texLive = texlive.combine { inherit (texlive) scheme-small libertinus-fonts; }; + texLive = texliveSmall.withPackages (ps: [ ps.libertinus-fonts ]); text = '' \documentclass{article} \usepackage{fontspec} @@ -61,7 +61,7 @@ rec { chktex = runCommand "texlive-test-chktex" { nativeBuildInputs = [ - (texlive.combine { inherit (texlive) scheme-infraonly chktex; }) + (texlive.withPackages (ps: [ ps.chktex ])) ]; input = builtins.toFile "chktex-sample.tex" '' \documentclass{article} @@ -77,7 +77,7 @@ rec { dvipng = lib.recurseIntoAttrs { # https://github.com/NixOS/nixpkgs/issues/75605 basic = runCommand "texlive-test-dvipng-basic" { - nativeBuildInputs = [ file texlive.combined.scheme-medium ]; + nativeBuildInputs = [ file texliveMedium ]; input = fetchurl { name = "test_dvipng.tex"; url = "http://git.savannah.nongnu.org/cgit/dvipng.git/plain/test_dvipng.tex?id=b872753590a18605260078f56cbd6f28d39dc035"; @@ -99,7 +99,7 @@ rec { # test dvipng's limited capability to render postscript specials via GS ghostscript = runCommand "texlive-test-ghostscript" { - nativeBuildInputs = [ file (texlive.combine { inherit (texlive) scheme-small dvipng; }) ]; + nativeBuildInputs = [ file (texliveSmall.withPackages (ps: [ ps.dvipng ])) ]; input = builtins.toFile "postscript-sample.tex" '' \documentclass{minimal} \begin{document} @@ -140,7 +140,7 @@ rec { # https://github.com/NixOS/nixpkgs/issues/75070 dvisvgm = runCommand "texlive-test-dvisvgm" { - nativeBuildInputs = [ file texlive.combined.scheme-medium ]; + nativeBuildInputs = [ file texliveMedium ]; input = builtins.toFile "dvisvgm-sample.tex" '' \documentclass{article} \begin{document} @@ -166,10 +166,7 @@ rec { texdoc = runCommand "texlive-test-texdoc" { nativeBuildInputs = [ - (texlive.combine { - inherit (texlive) scheme-infraonly luatex texdoc; - pkgFilter = pkg: lib.elem pkg.tlType [ "run" "bin" "doc" ]; - }) + (texlive.withPackages (ps: with ps; [ luatex ps.texdoc ps.texdoc.texdoc ])) ]; } '' texdoc --version @@ -214,7 +211,7 @@ rec { }; # check that all languages are available, including synonyms - allLanguages = let hyphenBase = lib.head texlive.hyphen-base.pkgs; texLive = texlive.combined.scheme-full; in + allLanguages = let hyphenBase = texlive.pkgs.hyphen-base; texLive = texliveFull; in lib.recurseIntoAttrs { # language.def etex = mkTeXTest { @@ -289,9 +286,9 @@ rec { # test that language files are generated as expected hyphen-base = runCommand "texlive-test-hyphen-base" { - hyphenBase = lib.head texlive.hyphen-base.pkgs; - schemeFull = texlive.combined.scheme-full; - schemeInfraOnly = texlive.combined.scheme-infraonly; + hyphenBase = texlive.pkgs.hyphen-base; + schemeFull = texliveFull; + schemeInfraOnly = texliveInfraOnly; } '' mkdir -p "$out"/{scheme-infraonly,scheme-full} @@ -322,8 +319,8 @@ rec { # test that fmtutil.cnf is fully regenerated on scheme-full fmtutilCnf = runCommand "texlive-test-fmtutil.cnf" { - kpathsea = lib.head texlive.kpathsea.pkgs; - schemeFull = texlive.combined.scheme-full; + kpathsea = texlive.pkgs.kpathsea.tex; + schemeFull = texliveFull; } '' mkdir -p "$out" @@ -335,7 +332,7 @@ rec { # verify that the restricted mode gets enabled when # needed (detected by checking if it disallows --gscmd) repstopdf = runCommand "texlive-test-repstopdf" { - nativeBuildInputs = [ (texlive.combine { inherit (texlive) scheme-infraonly epstopdf; }) ]; + nativeBuildInputs = [ (texlive.withPackages (ps: [ ps.epstopdf ])) ]; } '' ! (epstopdf --gscmd echo /dev/null 2>&1 || true) | grep forbidden >/dev/null (repstopdf --gscmd echo /dev/null 2>&1 || true) | grep forbidden >/dev/null @@ -345,7 +342,7 @@ rec { # verify that the restricted mode gets enabled when # needed (detected by checking if it disallows --gscmd) rpdfcrop = runCommand "texlive-test-rpdfcrop" { - nativeBuildInputs = [ (texlive.combine { inherit (texlive) scheme-infraonly pdfcrop; }) ]; + nativeBuildInputs = [ (texlive.withPackages (ps: [ ps.pdfcrop ])) ]; } '' ! (pdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null (rpdfcrop --gscmd echo $(command -v pdfcrop) 2>&1 || true) | grep 'restricted mode' >/dev/null @@ -454,14 +451,13 @@ rec { ''; # link all binaries in single derivation - allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive))); - binPackages = lib.filter (p: p.tlType == "bin") allPackages; + binPackages = lib.catAttrs "out" (lib.attrValues texlive.pkgs); binaries = buildEnv { name = "texlive-binaries"; paths = binPackages; }; in runCommand "texlive-test-binaries" { inherit binaries contextTestTex latexTestTex texTestTex; - texliveScheme = texlive.combined.scheme-full; + texliveScheme = texliveFull; } '' loadables="$(command -v bash)" @@ -565,8 +561,7 @@ rec { # check that all scripts have a Nix shebang shebangs = let - allPackages = with lib; concatLists (catAttrs "pkgs" (filter isAttrs (attrValues texlive))); - binPackages = lib.filter (p: p.tlType == "bin") allPackages; + binPackages = lib.catAttrs "out" (lib.attrValues texlive.pkgs); in runCommand "texlive-test-shebangs" { } ('' @@ -615,7 +610,7 @@ rec { correctLicenses = scheme: builtins.foldl' (acc: pkg: concatLicenses acc (lib.toList (pkg.meta.license or []))) [] - scheme.passthru.packages; + scheme.passthru.requiredTeXPackages; correctLicensesAttrNames = scheme: lib.sort lt (map licenseToAttrName (correctLicenses scheme)); @@ -625,7 +620,7 @@ rec { (savedLicensesAttrNames scheme) != (correctLicensesAttrNames scheme); incorrectSchemes = lib.filterAttrs (n: hasLicenseMismatch) - texlive.combined; + (texlive.combined // texlive.schemes); prettyPrint = name: scheme: '' license info for ${name} is incorrect! Note that order is enforced. @@ -648,10 +643,13 @@ rec { # this is effectively an eval-time assertion, converted into a derivation for # ease of testing fixedHashes = with lib; let - combine = findFirst (p: (head p.pkgs).pname == "combine") { pkgs = []; } (head texlive.collection-latexextra.pkgs).tlDeps; - all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs; - fods = filter (p: isDerivation p && p.tlType != "bin") all; - errorText = concatMapStrings (p: optionalString (! p ? outputHash) "${p.pname + optionalString (p.tlType != "run") ("." + p.tlType)} does not have a fixed output hash\n") fods; + fods = lib.concatMap + (p: lib.optional (p ? tex && isDerivation p.tex) p.tex + ++ lib.optional (p ? texdoc) p.texdoc + ++ lib.optional (p ? texsource) p.texsource + ++ lib.optional (p ? tlpkg) p.tlpkg) + (attrValues texlive.pkgs); + errorText = concatMapStrings (p: optionalString (! p ? outputHash) "${p.pname}-${p.tlOutputName} does not have a fixed output hash\n") fods; in runCommand "texlive-test-fixed-hashes" { inherit errorText; passAsFile = [ "errorText" ]; diff --git a/pkgs/test/vim/default.nix b/pkgs/test/vim/default.nix index 4d8e59a306a..33e1e551d4f 100644 --- a/pkgs/test/vim/default.nix +++ b/pkgs/test/vim/default.nix @@ -3,7 +3,7 @@ , pkgs }: let - inherit (vimUtils) buildVimPluginFrom2Nix; + inherit (vimUtils) buildVimPlugin; packages.myVimPackage.start = with vimPlugins; [ vim-nix ]; |