summary refs log tree commit diff
path: root/pkgs
diff options
context:
space:
mode:
authorSilvan Mosberger <silvan.mosberger@tweag.io>2023-08-23 04:22:41 +0200
committerSilvan Mosberger <silvan.mosberger@tweag.io>2023-08-29 16:17:54 +0200
commit271eb0299503892944986eb381b79ec09ea2f2a4 (patch)
tree0548e8ae5c981fc09f15365d95a05e63d790bcb1 /pkgs
parent87c5a6a84fcb94fd52507c24bb31c3b844775190 (diff)
downloadnixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar.gz
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar.bz2
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar.lz
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar.xz
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.tar.zst
nixpkgs-271eb0299503892944986eb381b79ec09ea2f2a4.zip
pkgs/test/nixpkgs-check-by-name: init
Adds an internal CLI tool to verify Nixpkgs to conform to RFC 140.
See pkgs/test/nixpkgs-check-by-name/README.md for more information.
Diffstat (limited to 'pkgs')
-rw-r--r--pkgs/test/default.nix2
-rw-r--r--pkgs/test/nixpkgs-check-by-name/.envrc1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/.gitignore2
-rw-r--r--pkgs/test/nixpkgs-check-by-name/Cargo.lock528
-rw-r--r--pkgs/test/nixpkgs-check-by-name/Cargo.toml16
-rw-r--r--pkgs/test/nixpkgs-check-by-name/README.md82
-rw-r--r--pkgs/test/nixpkgs-check-by-name/default.nix38
-rw-r--r--pkgs/test/nixpkgs-check-by-name/shell.nix6
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/eval.nix59
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/eval.rs124
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/main.rs133
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/references.rs184
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/structure.rs152
-rw-r--r--pkgs/test/nixpkgs-check-by-name/src/utils.rs72
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix4
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/pkgs/by-name/fo/foo/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foO/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foo/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/pkgs/by-name/aa/FOO/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/pkgs/by-name/fo/fo@/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/.git-keep0
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/pkgs/by-name/fo/foo/.git-keep0
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix101
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-attrs/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-attrs/pkgs/by-name/no/nonDerivation/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-derivation/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/non-derivation/pkgs/by-name/no/nonDerivation/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/one-letter/pkgs/by-name/a/a/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-different-file/all-packages.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-different-file/pkgs/by-name/no/nonDerivation/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-different-file/someDrv.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/pkgs/by-name/no/nonDerivation/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/override-no-file/pkgs/by-name/no/nonDerivation/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/pkgs/by-name/fo/foo0
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/pkgs/by-name/fo/foo/package.nix/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/pkgs/by-name/aa/aa/package.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-escape/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-escape/pkgs/by-name/aa/aa/package.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/pkgs/by-name/aa/aa/package.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/invalid.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/pkgs/by-name/aa/aa/package.nix3
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/dir/default.nix2
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file0
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file.nix2
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/package.nix5
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/shard-file/expected1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/shard-file/pkgs/by-name/fo0
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/success/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/success/pkgs/by-name/fo/foo/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/expected1
l---------pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/pkgs/by-name/fo/foo/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/someDrv.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/expected1
l---------pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/someDrv.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/uppercase/pkgs/by-name/fo/FOO/package.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix1
-rw-r--r--pkgs/test/nixpkgs-check-by-name/tests/with-readme/pkgs/by-name/README.md0
102 files changed, 1604 insertions, 0 deletions
diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix
index 6bfa1c4393c..b9d75c790da 100644
--- a/pkgs/test/default.nix
+++ b/pkgs/test/default.nix
@@ -93,4 +93,6 @@ with pkgs;
   };
 
   pkgs-lib = recurseIntoAttrs (import ../pkgs-lib/tests { inherit pkgs; });
+
+  nixpkgs-check-by-name = callPackage ./nixpkgs-check-by-name { };
 }
diff --git a/pkgs/test/nixpkgs-check-by-name/.envrc b/pkgs/test/nixpkgs-check-by-name/.envrc
new file mode 100644
index 00000000000..1d953f4bd73
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/.envrc
@@ -0,0 +1 @@
+use nix
diff --git a/pkgs/test/nixpkgs-check-by-name/.gitignore b/pkgs/test/nixpkgs-check-by-name/.gitignore
new file mode 100644
index 00000000000..75e92a90898
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/.gitignore
@@ -0,0 +1,2 @@
+target
+.direnv
diff --git a/pkgs/test/nixpkgs-check-by-name/Cargo.lock b/pkgs/test/nixpkgs-check-by-name/Cargo.lock
new file mode 100644
index 00000000000..3859d2b6e97
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/Cargo.lock
@@ -0,0 +1,528 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anstream"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+
+[[package]]
+name = "cc"
+version = "1.0.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "4.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+ "once_cell",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
+[[package]]
+name = "colored"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6"
+dependencies = [
+ "is-terminal",
+ "lazy_static",
+ "windows-sys",
+]
+
+[[package]]
+name = "countme"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
+
+[[package]]
+name = "errno"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.147"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "memoffset"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "nixpkgs-check-by-name"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "colored",
+ "lazy_static",
+ "regex",
+ "rnix",
+ "rowan",
+ "serde",
+ "serde_json",
+ "tempfile",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "regex"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+
+[[package]]
+name = "rnix"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb35cedbeb70e0ccabef2a31bcff0aebd114f19566086300b8f42c725fc2cb5f"
+dependencies = [
+ "rowan",
+]
+
+[[package]]
+name = "rowan"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64449cfef9483a475ed56ae30e2da5ee96448789fb2aa240a04beb6a055078bf"
+dependencies = [
+ "countme",
+ "hashbrown",
+ "memoffset",
+ "rustc-hash",
+ "text-size",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustix"
+version = "0.38.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
+dependencies = [
+ "bitflags 2.4.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
+[[package]]
+name = "serde"
+version = "1.0.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f5db24220c009de9bd45e69fb2938f4b6d2df856aa9304ce377b3180f83b7c1"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ad697f7e0b65af4983a4ce8f56ed5b357e8d3c36651bf6a7e13639c17b8e670"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.105"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "2.0.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "redox_syscall",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "text-size"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
diff --git a/pkgs/test/nixpkgs-check-by-name/Cargo.toml b/pkgs/test/nixpkgs-check-by-name/Cargo.toml
new file mode 100644
index 00000000000..003aab6d584
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "nixpkgs-check-by-name"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+rnix = "0.11.0"
+rowan = "0.15.0"
+regex = "1.9.3"
+clap = { version = "4.3.23", features = ["derive"] }
+serde_json = "1.0.105"
+tempfile = "3.8.0"
+serde = { version = "1.0.185", features = ["derive"] }
+anyhow = "1.0"
+lazy_static = "1.4.0"
+colored = "2.0.4"
diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md
new file mode 100644
index 00000000000..16536154a26
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/README.md
@@ -0,0 +1,82 @@
+# Nixpkgs pkgs/by-name checker
+
+This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory once introduced.
+This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140).
+
+## API
+
+This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately.
+
+- Command line: `nixpkgs-check-by-name <NIXPKGS>`
+- Arguments:
+  - `<NIXPKGS>`: The path to the Nixpkgs to check
+- Exit code:
+  - `0`: If the [validation](#validity-checks) is successful
+  - `1`: If the [validation](#validity-checks) is not successful
+  - `2`: If an unexpected I/O error occurs
+- Standard error:
+  - Informative messages
+  - Error messages if validation is not successful
+
+## Validity checks
+
+These checks are performed by this tool:
+
+### File structure checks
+- `pkgs/by-name` must only contain subdirectories of the form `${shard}/${name}`, called _package directories_.
+- The `name`'s of package directories must be unique when lowercased
+- `name` is a string only consisting of the ASCII characters `a-z`, `A-Z`, `0-9`, `-` or `_`.
+- `shard` is the lowercased first two letters of `name`, expressed in Nix: `shard = toLower (substring 0 2 name)`.
+- Each package directory must contain a `package.nix` file and may contain arbitrary other files.
+
+### Nix parser checks
+- Each package directory must not refer to files outside itself using symlinks or Nix path expressions.
+
+### Nix evaluation checks
+- `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`.
+- `pkgs.lib.isDerivation pkgs.${name}` is `true`.
+
+## Development
+
+Enter the development environment in this directory either automatically with `direnv` or with
+```
+nix-shell
+```
+
+Then use `cargo`:
+```
+cargo build
+cargo test
+cargo fmt
+cargo clippy
+```
+
+## Tests
+
+Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs with these files:
+- `default.nix`:
+  Always contains
+  ```nix
+  import ../mock-nixpkgs.nix { root = ./.; }
+  ```
+  which makes
+  ```
+  nix-instantiate <subdir> --eval -A <attr> --arg overlays <overlays>
+  ```
+  work very similarly to the real Nixpkgs, just enough for the program to be able to test it.
+- `pkgs/by-name`:
+  The `pkgs/by-name` directory to check.
+
+- `all-packages.nix` (optional):
+  Contains an overlay of the form
+  ```nix
+  self: super: {
+    # ...
+  }
+  ```
+  allowing the simulation of package overrides to the real [`pkgs/top-level/all-packages.nix`](../../top-level/all-packages.nix`).
+  The default is an empty overlay.
+
+- `expected` (optional):
+  A file containing the expected standard output.
+  The default is expecting an empty standard output.
diff --git a/pkgs/test/nixpkgs-check-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/default.nix
new file mode 100644
index 00000000000..a997fc8612c
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/default.nix
@@ -0,0 +1,38 @@
+{
+  lib,
+  rustPlatform,
+  nix,
+  rustfmt,
+  clippy,
+  mkShell,
+}:
+let
+  package =
+    rustPlatform.buildRustPackage {
+      name = "nixpkgs-check-by-name";
+      src = lib.cleanSource ./.;
+      cargoLock.lockFile = ./Cargo.lock;
+      nativeBuildInputs = [
+        nix
+        rustfmt
+        clippy
+      ];
+      # Needed to make Nix evaluation work inside the nix build
+      preCheck = ''
+        export TEST_ROOT=$(pwd)/test-tmp
+        export NIX_CONF_DIR=$TEST_ROOT/etc
+        export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
+        export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
+        export NIX_STATE_DIR=$TEST_ROOT/var/nix
+        export NIX_STORE_DIR=$TEST_ROOT/store
+      '';
+      postCheck = ''
+        cargo fmt --check
+        cargo clippy -- -D warnings
+      '';
+      passthru.shell = mkShell {
+        inputsFrom = [ package ];
+      };
+    };
+in
+package
diff --git a/pkgs/test/nixpkgs-check-by-name/shell.nix b/pkgs/test/nixpkgs-check-by-name/shell.nix
new file mode 100644
index 00000000000..33bcf45b8d0
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/shell.nix
@@ -0,0 +1,6 @@
+let
+  pkgs = import ../../.. {
+    config = {};
+    overlays = [];
+  };
+in pkgs.tests.nixpkgs-check-by-name.shell
diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
new file mode 100644
index 00000000000..7c0ae755215
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
@@ -0,0 +1,59 @@
+# Takes a path to nixpkgs and a path to the json-encoded list of attributes to check.
+# Returns an attribute set containing information on each requested attribute.
+# If the attribute is missing from Nixpkgs it's also missing from the result.
+#
+# The returned information is an attribute set with:
+# - call_package_path: The <path> from `<attr> = callPackage <path> { ... }`,
+#   or null if it's not defined as with callPackage, or if the <path> is not a path
+# - is_derivation: The result of `lib.isDerivation <attr>`
+{
+  attrsPath,
+  nixpkgsPath,
+}:
+let
+  attrs = builtins.fromJSON (builtins.readFile attrsPath);
+
+  # This overlay mocks callPackage to persist the path of the first argument
+  callPackageOverlay = self: super: {
+    callPackage = fn: args:
+      let
+        result = super.callPackage fn args;
+      in
+      if builtins.isAttrs result then
+        # If this was the last overlay to be applied, we could just only return the `_callPackagePath`,
+        # but that's not the case because stdenv has another overlays on top of user-provided ones.
+        # So to not break the stdenv build we need to return the mostly proper result here
+        result // {
+          _callPackagePath = fn;
+        }
+      else
+        # It's very rare that callPackage doesn't return an attribute set, but it can occur.
+        {
+          _callPackagePath = fn;
+        };
+  };
+
+  pkgs = import nixpkgsPath {
+    # Don't let the users home directory influence this result
+    config = { };
+    overlays = [ callPackageOverlay ];
+  };
+
+  attrInfo = attr: {
+    # These names are used by the deserializer on the Rust side
+    call_package_path =
+      if pkgs.${attr} ? _callPackagePath && builtins.isPath pkgs.${attr}._callPackagePath then
+        toString pkgs.${attr}._callPackagePath
+      else
+        null;
+    is_derivation = pkgs.lib.isDerivation pkgs.${attr};
+  };
+
+  attrInfos = builtins.listToAttrs (map (name: {
+    inherit name;
+    value = attrInfo name;
+  }) attrs);
+
+in
+# Filter out attributes not in Nixpkgs
+builtins.intersectAttrs pkgs attrInfos
diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
new file mode 100644
index 00000000000..d084642ffe7
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -0,0 +1,124 @@
+use crate::structure;
+use crate::utils::ErrorWriter;
+use std::path::Path;
+
+use anyhow::Context;
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::io;
+use std::path::PathBuf;
+use std::process;
+use tempfile::NamedTempFile;
+
+/// Attribute set of this structure is returned by eval.nix
+#[derive(Deserialize)]
+struct AttributeInfo {
+    call_package_path: Option<PathBuf>,
+    is_derivation: bool,
+}
+
+const EXPR: &str = include_str!("eval.nix");
+
+/// Check that the Nixpkgs attribute values corresponding to the packages in pkgs/by-name are
+/// of the form `callPackage <package_file> { ... }`.
+/// See the `eval.nix` file for how this is achieved on the Nix side
+pub fn check_values<W: io::Write>(
+    error_writer: &mut ErrorWriter<W>,
+    nixpkgs: &structure::Nixpkgs,
+    eval_accessible_paths: Vec<&Path>,
+) -> anyhow::Result<()> {
+    // Write the list of packages we need to check into a temporary JSON file.
+    // This can then get read by the Nix evaluation.
+    let attrs_file = NamedTempFile::new().context("Failed to create a temporary file")?;
+    serde_json::to_writer(&attrs_file, &nixpkgs.package_names).context(format!(
+        "Failed to serialise the package names to the temporary path {}",
+        attrs_file.path().display()
+    ))?;
+
+    // With restrict-eval, only paths in NIX_PATH can be accessed, so we explicitly specify the
+    // ones needed needed
+
+    let mut command = process::Command::new("nix-instantiate");
+    command
+        // Inherit stderr so that error messages always get shown
+        .stderr(process::Stdio::inherit())
+        // Clear NIX_PATH to be sure it doesn't influence the result
+        .env_remove("NIX_PATH")
+        .args([
+            "--eval",
+            "--json",
+            "--strict",
+            "--readonly-mode",
+            "--restrict-eval",
+            "--show-trace",
+            "--expr",
+            EXPR,
+        ])
+        // Pass the path to the attrs_file as an argument and add it to the NIX_PATH so it can be
+        // accessed in restrict-eval mode
+        .args(["--arg", "attrsPath"])
+        .arg(attrs_file.path())
+        .arg("-I")
+        .arg(attrs_file.path())
+        // Same for the nixpkgs to test
+        .args(["--arg", "nixpkgsPath"])
+        .arg(&nixpkgs.path)
+        .arg("-I")
+        .arg(&nixpkgs.path);
+
+    // Also add extra paths that need to be accessible
+    for path in eval_accessible_paths {
+        command.arg("-I");
+        command.arg(path);
+    }
+
+    let result = command
+        .output()
+        .context(format!("Failed to run command {command:?}"))?;
+
+    if !result.status.success() {
+        anyhow::bail!("Failed to run command {command:?}");
+    }
+    // Parse the resulting JSON value
+    let actual_files: HashMap<String, AttributeInfo> = serde_json::from_slice(&result.stdout)
+        .context(format!(
+            "Failed to deserialise {}",
+            String::from_utf8_lossy(&result.stdout)
+        ))?;
+
+    for package_name in &nixpkgs.package_names {
+        let relative_package_file = structure::Nixpkgs::relative_file_for_package(package_name);
+        let absolute_package_file = nixpkgs.path.join(&relative_package_file);
+
+        if let Some(attribute_info) = actual_files.get(package_name) {
+            let is_expected_file =
+                if let Some(call_package_path) = &attribute_info.call_package_path {
+                    absolute_package_file == *call_package_path
+                } else {
+                    false
+                };
+
+            if !is_expected_file {
+                error_writer.write(&format!(
+                    "pkgs.{package_name}: This attribute is not defined as `pkgs.callPackage {} {{ ... }}`.",
+                    relative_package_file.display()
+                ))?;
+                continue;
+            }
+
+            if !attribute_info.is_derivation {
+                error_writer.write(&format!(
+                    "pkgs.{package_name}: This attribute defined by {} is not a derivation",
+                    relative_package_file.display()
+                ))?;
+            }
+        } else {
+            error_writer.write(&format!(
+                "pkgs.{package_name}: This attribute is not defined but it should be defined automatically as {}",
+                relative_package_file.display()
+            ))?;
+            continue;
+        }
+    }
+    Ok(())
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs
new file mode 100644
index 00000000000..1f4a7b99a99
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs
@@ -0,0 +1,133 @@
+mod eval;
+mod references;
+mod structure;
+mod utils;
+
+use anyhow::Context;
+use clap::Parser;
+use colored::Colorize;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+use structure::Nixpkgs;
+use utils::ErrorWriter;
+
+/// Program to check the validity of pkgs/by-name
+#[derive(Parser, Debug)]
+#[command(about)]
+struct Args {
+    /// Path to nixpkgs
+    nixpkgs: PathBuf,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+    match check_nixpkgs(&args.nixpkgs, vec![], &mut io::stderr()) {
+        Ok(true) => {
+            eprintln!("{}", "Validated successfully".green());
+            ExitCode::SUCCESS
+        }
+        Ok(false) => {
+            eprintln!("{}", "Validation failed, see above errors".yellow());
+            ExitCode::from(1)
+        }
+        Err(e) => {
+            eprintln!("{} {:#}", "I/O error: ".yellow(), e);
+            ExitCode::from(2)
+        }
+    }
+}
+
+/// Checks whether the pkgs/by-name structure in Nixpkgs is valid.
+///
+/// # Arguments
+/// - `nixpkgs_path`: The path to the Nixpkgs to check
+/// - `eval_accessible_paths`:
+///   Extra paths that need to be accessible to evaluate Nixpkgs using `restrict-eval`.
+///   This is used to allow the tests to access the mock-nixpkgs.nix file
+/// - `error_writer`: An `io::Write` value to write validation errors to, if any.
+///
+/// # Return value
+/// - `Err(e)` if an I/O-related error `e` occurred.
+/// - `Ok(false)` if the structure is invalid, all the structural errors have been written to `error_writer`.
+/// - `Ok(true)` if the structure is valid, nothing will have been written to `error_writer`.
+pub fn check_nixpkgs<W: io::Write>(
+    nixpkgs_path: &Path,
+    eval_accessible_paths: Vec<&Path>,
+    error_writer: &mut W,
+) -> anyhow::Result<bool> {
+    let nixpkgs_path = nixpkgs_path.canonicalize().context(format!(
+        "Nixpkgs path {} could not be resolved",
+        nixpkgs_path.display()
+    ))?;
+
+    // Wraps the error_writer to print everything in red, and tracks whether anything was printed
+    // at all. Later used to figure out if the structure was valid or not.
+    let mut error_writer = ErrorWriter::new(error_writer);
+
+    if !nixpkgs_path.join(structure::BASE_SUBPATH).exists() {
+        eprintln!(
+            "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.",
+            structure::BASE_SUBPATH
+        );
+    } else {
+        let nixpkgs = Nixpkgs::new(&nixpkgs_path, &mut error_writer)?;
+
+        if error_writer.empty {
+            // Only if we could successfully parse the structure, we do the semantic checks
+            eval::check_values(&mut error_writer, &nixpkgs, eval_accessible_paths)?;
+            references::check_references(&mut error_writer, &nixpkgs)?;
+        }
+    }
+    Ok(error_writer.empty)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::check_nixpkgs;
+    use anyhow::Context;
+    use std::env;
+    use std::fs;
+    use std::path::PathBuf;
+
+    #[test]
+    fn test_cases() -> anyhow::Result<()> {
+        let extra_nix_path = PathBuf::from("tests/mock-nixpkgs.nix");
+
+        // We don't want coloring to mess up the tests
+        env::set_var("NO_COLOR", "1");
+
+        for entry in PathBuf::from("tests").read_dir()? {
+            let entry = entry?;
+            let path = entry.path();
+            let name = entry.file_name().to_string_lossy().into_owned();
+
+            if !entry.path().is_dir() {
+                continue;
+            }
+
+            // This test explicitly makes sure we don't add files that would cause problems on
+            // Darwin, so we cannot test it on Darwin itself
+            #[cfg(not(target_os = "linux"))]
+            if name == "case-sensitive-duplicate-package" {
+                continue;
+            }
+
+            let mut writer = vec![];
+            check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)
+                .context(format!("Failed test case {name}"))?;
+
+            let actual_errors = String::from_utf8_lossy(&writer);
+            let expected_errors =
+                fs::read_to_string(path.join("expected")).unwrap_or(String::new());
+
+            if actual_errors != expected_errors {
+                panic!(
+                    "Failed test case {name}, expected these errors:\n\n{}\n\nbut got these:\n\n{}",
+                    expected_errors, actual_errors
+                );
+            }
+        }
+        Ok(())
+    }
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/src/references.rs b/pkgs/test/nixpkgs-check-by-name/src/references.rs
new file mode 100644
index 00000000000..16dc60729c4
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/references.rs
@@ -0,0 +1,184 @@
+use crate::structure::Nixpkgs;
+use crate::utils;
+use crate::utils::{ErrorWriter, LineIndex};
+
+use anyhow::Context;
+use rnix::{Root, SyntaxKind::NODE_PATH};
+use std::ffi::OsStr;
+use std::fs::read_to_string;
+use std::io;
+use std::path::{Path, PathBuf};
+
+/// Small helper so we don't need to pass in the same arguments to all functions
+struct PackageContext<'a, W: io::Write> {
+    error_writer: &'a mut ErrorWriter<W>,
+    /// The package directory relative to Nixpkgs, such as `pkgs/by-name/fo/foo`
+    relative_package_dir: &'a PathBuf,
+    /// The absolute package directory
+    absolute_package_dir: &'a PathBuf,
+}
+
+/// Check that every package directory in pkgs/by-name doesn't link to outside that directory.
+/// Both symlinks and Nix path expressions are checked.
+pub fn check_references<W: io::Write>(
+    error_writer: &mut ErrorWriter<W>,
+    nixpkgs: &Nixpkgs,
+) -> anyhow::Result<()> {
+    // Check the directories for each package separately
+    for package_name in &nixpkgs.package_names {
+        let relative_package_dir = Nixpkgs::relative_dir_for_package(package_name);
+        let mut context = PackageContext {
+            error_writer,
+            relative_package_dir: &relative_package_dir,
+            absolute_package_dir: &nixpkgs.path.join(&relative_package_dir),
+        };
+
+        // The empty argument here is the subpath under the package directory to check
+        // An empty one means the package directory itself
+        check_path(&mut context, Path::new("")).context(format!(
+            "While checking the references in package directory {}",
+            relative_package_dir.display()
+        ))?;
+    }
+    Ok(())
+}
+
+/// Checks for a specific path to not have references outside
+fn check_path<W: io::Write>(context: &mut PackageContext<W>, subpath: &Path) -> anyhow::Result<()> {
+    let path = context.absolute_package_dir.join(subpath);
+
+    if path.is_symlink() {
+        // Check whether the symlink resolves to outside the package directory
+        match path.canonicalize() {
+            Ok(target) => {
+                // No need to handle the case of it being inside the directory, since we scan through the
+                // entire directory recursively anyways
+                if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) {
+                    context.error_writer.write(&format!(
+                        "{}: Path {} is a symlink pointing to a path outside the directory of that package.",
+                        context.relative_package_dir.display(),
+                        subpath.display(),
+                    ))?;
+                }
+            }
+            Err(e) => {
+                context.error_writer.write(&format!(
+                    "{}: Path {} is a symlink which cannot be resolved: {e}.",
+                    context.relative_package_dir.display(),
+                    subpath.display(),
+                ))?;
+            }
+        }
+    } else if path.is_dir() {
+        // Recursively check each entry
+        for entry in utils::read_dir_sorted(&path)? {
+            let entry_subpath = subpath.join(entry.file_name());
+            check_path(context, &entry_subpath)
+                .context(format!("Error while recursing into {}", subpath.display()))?
+        }
+    } else if path.is_file() {
+        // Only check Nix files
+        if let Some(ext) = path.extension() {
+            if ext == OsStr::new("nix") {
+                check_nix_file(context, subpath).context(format!(
+                    "Error while checking Nix file {}",
+                    subpath.display()
+                ))?
+            }
+        }
+    } else {
+        // This should never happen, git doesn't support other file types
+        anyhow::bail!("Unsupported file type for path {}", subpath.display());
+    }
+    Ok(())
+}
+
+/// Check whether a nix file contains path expression references pointing outside the package
+/// directory
+fn check_nix_file<W: io::Write>(
+    context: &mut PackageContext<W>,
+    subpath: &Path,
+) -> anyhow::Result<()> {
+    let path = context.absolute_package_dir.join(subpath);
+    let parent_dir = path.parent().context(format!(
+        "Could not get parent of path {}",
+        subpath.display()
+    ))?;
+
+    let contents =
+        read_to_string(&path).context(format!("Could not read file {}", subpath.display()))?;
+
+    let root = Root::parse(&contents);
+    if let Some(error) = root.errors().first() {
+        context.error_writer.write(&format!(
+            "{}: File {} could not be parsed by rnix: {}",
+            context.relative_package_dir.display(),
+            subpath.display(),
+            error,
+        ))?;
+        return Ok(());
+    }
+
+    let line_index = LineIndex::new(&contents);
+
+    for node in root.syntax().descendants() {
+        // We're only interested in Path expressions
+        if node.kind() != NODE_PATH {
+            continue;
+        }
+
+        let text = node.text().to_string();
+        let line = line_index.line(node.text_range().start().into());
+
+        // Filters out ./foo/${bar}/baz
+        // TODO: We can just check ./foo
+        if node.children().count() != 0 {
+            context.error_writer.write(&format!(
+                "{}: File {} at line {line} contains the path expression \"{}\", which is not yet supported and may point outside the directory of that package.",
+                context.relative_package_dir.display(),
+                subpath.display(),
+                text
+            ))?;
+            continue;
+        }
+
+        // Filters out search paths like <nixpkgs>
+        if text.starts_with('<') {
+            context.error_writer.write(&format!(
+                "{}: File {} at line {line} contains the nix search path expression \"{}\" which may point outside the directory of that package.",
+                context.relative_package_dir.display(),
+                subpath.display(),
+                text
+            ))?;
+            continue;
+        }
+
+        // Resolves the reference of the Nix path
+        // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz`
+        match parent_dir.join(Path::new(&text)).canonicalize() {
+            Ok(target) => {
+                // Then checking if it's still in the package directory
+                // No need to handle the case of it being inside the directory, since we scan through the
+                // entire directory recursively anyways
+                if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) {
+                    context.error_writer.write(&format!(
+                        "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.",
+                        context.relative_package_dir.display(),
+                        subpath.display(),
+                        text,
+                    ))?;
+                }
+            }
+            Err(e) => {
+                context.error_writer.write(&format!(
+                    "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {e}.",
+                    context.relative_package_dir.display(),
+                    subpath.display(),
+                    text,
+                ))?;
+            }
+        };
+    }
+
+    Ok(())
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/src/structure.rs b/pkgs/test/nixpkgs-check-by-name/src/structure.rs
new file mode 100644
index 00000000000..ea80128e487
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/structure.rs
@@ -0,0 +1,152 @@
+use crate::utils;
+use crate::utils::ErrorWriter;
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::collections::HashMap;
+use std::io;
+use std::path::{Path, PathBuf};
+
+pub const BASE_SUBPATH: &str = "pkgs/by-name";
+pub const PACKAGE_NIX_FILENAME: &str = "package.nix";
+
+lazy_static! {
+    static ref SHARD_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_-]{1,2}$").unwrap();
+    static ref PACKAGE_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_-]+$").unwrap();
+}
+
+/// Contains information about the structure of the pkgs/by-name directory of a Nixpkgs
+pub struct Nixpkgs {
+    /// The path to nixpkgs
+    pub path: PathBuf,
+    /// The names of all packages declared in pkgs/by-name
+    pub package_names: Vec<String>,
+}
+
+impl Nixpkgs {
+    // Some utility functions for the basic structure
+
+    pub fn shard_for_package(package_name: &str) -> String {
+        package_name.to_lowercase().chars().take(2).collect()
+    }
+
+    pub fn relative_dir_for_shard(shard_name: &str) -> PathBuf {
+        PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}"))
+    }
+
+    pub fn relative_dir_for_package(package_name: &str) -> PathBuf {
+        Nixpkgs::relative_dir_for_shard(&Nixpkgs::shard_for_package(package_name))
+            .join(package_name)
+    }
+
+    pub fn relative_file_for_package(package_name: &str) -> PathBuf {
+        Nixpkgs::relative_dir_for_package(package_name).join(PACKAGE_NIX_FILENAME)
+    }
+}
+
+impl Nixpkgs {
+    /// Read the structure of a Nixpkgs directory, displaying errors on the writer.
+    /// May return early with I/O errors.
+    pub fn new<W: io::Write>(
+        path: &Path,
+        error_writer: &mut ErrorWriter<W>,
+    ) -> anyhow::Result<Nixpkgs> {
+        let base_dir = path.join(BASE_SUBPATH);
+
+        let mut package_names = Vec::new();
+
+        for shard_entry in utils::read_dir_sorted(&base_dir)? {
+            let shard_path = shard_entry.path();
+            let shard_name = shard_entry.file_name().to_string_lossy().into_owned();
+            let relative_shard_path = Nixpkgs::relative_dir_for_shard(&shard_name);
+
+            if shard_name == "README.md" {
+                // README.md is allowed to be a file and not checked
+                continue;
+            }
+
+            if !shard_path.is_dir() {
+                error_writer.write(&format!(
+                    "{}: This is a file, but it should be a directory.",
+                    relative_shard_path.display(),
+                ))?;
+                // we can't check for any other errors if it's a file, since there's no subdirectories to check
+                continue;
+            }
+
+            let shard_name_valid = SHARD_NAME_REGEX.is_match(&shard_name);
+            if !shard_name_valid {
+                error_writer.write(&format!(
+                    "{}: Invalid directory name \"{shard_name}\", must be at most 2 ASCII characters consisting of a-z, 0-9, \"-\" or \"_\".",
+                    relative_shard_path.display()
+                ))?;
+            }
+
+            let mut unique_package_names = HashMap::new();
+
+            for package_entry in utils::read_dir_sorted(&shard_path)? {
+                let package_path = package_entry.path();
+                let package_name = package_entry.file_name().to_string_lossy().into_owned();
+                let relative_package_dir =
+                    PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}/{package_name}"));
+
+                if !package_path.is_dir() {
+                    error_writer.write(&format!(
+                        "{}: This path is a file, but it should be a directory.",
+                        relative_package_dir.display(),
+                    ))?;
+                    continue;
+                }
+
+                if let Some(duplicate_package_name) =
+                    unique_package_names.insert(package_name.to_lowercase(), package_name.clone())
+                {
+                    error_writer.write(&format!(
+                        "{}: Duplicate case-sensitive package directories \"{duplicate_package_name}\" and \"{package_name}\".",
+                        relative_shard_path.display(),
+                    ))?;
+                }
+
+                let package_name_valid = PACKAGE_NAME_REGEX.is_match(&package_name);
+                if !package_name_valid {
+                    error_writer.write(&format!(
+                        "{}: Invalid package directory name \"{package_name}\", must be ASCII characters consisting of a-z, A-Z, 0-9, \"-\" or \"_\".",
+                        relative_package_dir.display(),
+                    ))?;
+                }
+
+                let correct_relative_package_dir = Nixpkgs::relative_dir_for_package(&package_name);
+                if relative_package_dir != correct_relative_package_dir {
+                    // Only show this error if we have a valid shard and package name
+                    // Because if one of those is wrong, you should fix that first
+                    if shard_name_valid && package_name_valid {
+                        error_writer.write(&format!(
+                            "{}: Incorrect directory location, should be {} instead.",
+                            relative_package_dir.display(),
+                            correct_relative_package_dir.display(),
+                        ))?;
+                    }
+                }
+
+                let package_nix_path = package_path.join(PACKAGE_NIX_FILENAME);
+                if !package_nix_path.exists() {
+                    error_writer.write(&format!(
+                        "{}: Missing required \"{PACKAGE_NIX_FILENAME}\" file.",
+                        relative_package_dir.display(),
+                    ))?;
+                } else if package_nix_path.is_dir() {
+                    error_writer.write(&format!(
+                        "{}: \"{PACKAGE_NIX_FILENAME}\" must be a file.",
+                        relative_package_dir.display(),
+                    ))?;
+                }
+
+                package_names.push(package_name.clone());
+            }
+        }
+
+        Ok(Nixpkgs {
+            path: path.to_owned(),
+            package_names,
+        })
+    }
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/src/utils.rs b/pkgs/test/nixpkgs-check-by-name/src/utils.rs
new file mode 100644
index 00000000000..325c736eca9
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/src/utils.rs
@@ -0,0 +1,72 @@
+use anyhow::Context;
+use colored::Colorize;
+use std::fs;
+use std::io;
+use std::path::Path;
+
+/// Deterministic file listing so that tests are reproducible
+pub fn read_dir_sorted(base_dir: &Path) -> anyhow::Result<Vec<fs::DirEntry>> {
+    let listing = base_dir
+        .read_dir()
+        .context(format!("Could not list directory {}", base_dir.display()))?;
+    let mut shard_entries = listing
+        .collect::<io::Result<Vec<_>>>()
+        .context(format!("Could not list directory {}", base_dir.display()))?;
+    shard_entries.sort_by_key(|entry| entry.file_name());
+    Ok(shard_entries)
+}
+
+/// A simple utility for calculating the line for a string offset.
+/// This doesn't do any Unicode handling, though that probably doesn't matter
+/// because newlines can't split up Unicode characters. Also this is only used
+/// for error reporting
+pub struct LineIndex {
+    /// Stores the indices of newlines
+    newlines: Vec<usize>,
+}
+
+impl LineIndex {
+    pub fn new(s: &str) -> LineIndex {
+        let mut newlines = vec![];
+        let mut index = 0;
+        // Iterates over all newline-split parts of the string, adding the index of the newline to
+        // the vec
+        for split in s.split_inclusive('\n') {
+            index += split.len();
+            newlines.push(index);
+        }
+        LineIndex { newlines }
+    }
+
+    /// Returns the line number for a string index
+    pub fn line(&self, index: usize) -> usize {
+        match self.newlines.binary_search(&index) {
+            // +1 because lines are 1-indexed
+            Ok(x) => x + 1,
+            Err(x) => x + 1,
+        }
+    }
+}
+
+/// A small wrapper around a generic io::Write specifically for errors:
+/// - Print everything in red to signal it's an error
+/// - Keep track of whether anything was printed at all, so that
+///   it can be queried whether any errors were encountered at all
+pub struct ErrorWriter<W> {
+    pub writer: W,
+    pub empty: bool,
+}
+
+impl<W: io::Write> ErrorWriter<W> {
+    pub fn new(writer: W) -> ErrorWriter<W> {
+        ErrorWriter {
+            writer,
+            empty: true,
+        }
+    }
+
+    pub fn write(&mut self, string: &str) -> io::Result<()> {
+        self.empty = false;
+        writeln!(self.writer, "{}", string.red())
+    }
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix
new file mode 100644
index 00000000000..793dfabd655
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix
@@ -0,0 +1,4 @@
+args:
+builtins.removeAttrs
+  (import ../mock-nixpkgs.nix { root = ./.; } args)
+  [ "foo" ]
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/expected b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/expected
new file mode 100644
index 00000000000..fff17c6c7cd
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/expected
@@ -0,0 +1 @@
+pkgs.foo: This attribute is not defined but it should be defined automatically as pkgs/by-name/fo/foo/package.nix
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/expected b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/expected
new file mode 100644
index 00000000000..e3928858221
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo: Duplicate case-sensitive package directories "foO" and "foo".
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foO/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foO/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foO/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/case-sensitive-duplicate-package/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/expected b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/expected
new file mode 100644
index 00000000000..3627368c0ef
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/FOO: Incorrect directory location, should be pkgs/by-name/fo/FOO instead.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/pkgs/by-name/aa/FOO/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/pkgs/by-name/aa/FOO/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/pkgs/by-name/aa/FOO/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/expected b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/expected
new file mode 100644
index 00000000000..8c8eafdcb3d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/fo@: Invalid package directory name "fo@", must be ASCII characters consisting of a-z, A-Z, 0-9, "-" or "_".
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/pkgs/by-name/fo/fo@/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/pkgs/by-name/fo/fo@/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/pkgs/by-name/fo/fo@/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/expected b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/expected
new file mode 100644
index 00000000000..248aa887796
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/expected
@@ -0,0 +1 @@
+pkgs/by-name/A: Invalid directory name "A", must be at most 2 ASCII characters consisting of a-z, 0-9, "-" or "_".
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/.git-keep b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/.git-keep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/.git-keep
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/pkgs/by-name/A/A/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/expected b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/expected
new file mode 100644
index 00000000000..ce1afcbf2d3
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/foo: Missing required "package.nix" file.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/pkgs/by-name/fo/foo/.git-keep b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/pkgs/by-name/fo/foo/.git-keep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/pkgs/by-name/fo/foo/.git-keep
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix
new file mode 100644
index 00000000000..50ad6761754
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix
@@ -0,0 +1,101 @@
+/*
+This file returns a mocked version of Nixpkgs' default.nix for testing purposes.
+It does not depend on Nixpkgs itself for the sake of simplicity.
+
+It takes one attribute as an argument:
+- `root`: The root of Nixpkgs to read other files from, including:
+  - `./pkgs/by-name`: The `pkgs/by-name` directory to test
+  - `./all-packages.nix`: A file containing an overlay to mirror the real `pkgs/top-level/all-packages.nix`.
+    This allows adding overrides on top of the auto-called packages in `pkgs/by-name`.
+
+It returns a Nixpkgs-like function that can be auto-called and evaluates to an attribute set.
+*/
+{
+  root,
+}:
+# The arguments for the Nixpkgs function
+{
+  # Passed by the checker to modify `callPackage`
+  overlays ? [],
+  # Passed by the checker to make sure a real Nixpkgs isn't influenced by impurities
+  config ? {},
+}:
+let
+
+  # Simplified versions of lib functions
+  lib = {
+    fix = f: let x = f x; in x;
+
+    extends = overlay: f: final:
+      let
+        prev = f final;
+      in
+      prev // overlay final prev;
+
+    callPackageWith = autoArgs: fn: args:
+      let
+        f = if builtins.isFunction fn then fn else import fn;
+        fargs = builtins.functionArgs f;
+        allArgs = builtins.intersectAttrs fargs autoArgs // args;
+      in
+      f allArgs;
+
+    isDerivation = value: value.type or null == "derivation";
+  };
+
+  # The base fixed-point function to populate the resulting attribute set
+  pkgsFun = self: {
+    inherit lib;
+    callPackage = lib.callPackageWith self;
+    someDrv = { type = "derivation"; };
+  };
+
+  baseDirectory = root + "/pkgs/by-name";
+
+  # Generates { <name> = <file>; } entries mapping package names to their `package.nix` files in `pkgs/by-name`.
+  # Could be more efficient, but this is only for testing.
+  autoCalledPackageFiles =
+    let
+      entries = builtins.readDir baseDirectory;
+
+      namesForShard = shard:
+        if entries.${shard} != "directory" then
+          # Only README.md is allowed to be a file, but it's not this code's job to check for that
+          { }
+        else
+          builtins.mapAttrs
+            (name: _: baseDirectory + "/${shard}/${name}/package.nix")
+            (builtins.readDir (baseDirectory + "/${shard}"));
+
+    in
+    builtins.foldl'
+      (acc: el: acc // el)
+      { }
+      (map namesForShard (builtins.attrNames entries));
+
+  # Turns autoCalledPackageFiles into an overlay that `callPackage`'s all of them
+  autoCalledPackages = self: super:
+    builtins.mapAttrs (name: file:
+      self.callPackage file { }
+    ) autoCalledPackageFiles;
+
+  # A list optionally containing the `all-packages.nix` file from the test case as an overlay
+  optionalAllPackagesOverlay =
+    if builtins.pathExists (root + "/all-packages.nix") then
+      [ (import (root + "/all-packages.nix")) ]
+    else
+      [ ];
+
+  # All the overlays in the right order, including the user-supplied ones
+  allOverlays =
+    [
+      autoCalledPackages
+    ]
+    ++ optionalAllPackagesOverlay
+    ++ overlays;
+
+  # Apply all the overlays in order to the base fixed-point function pkgsFun
+  f = builtins.foldl' (f: overlay: lib.extends overlay f) pkgsFun allOverlays;
+in
+# Evaluate the fixed-point
+lib.fix f
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/expected b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/expected
new file mode 100644
index 00000000000..e6c92379010
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/expected
@@ -0,0 +1 @@
+pkgs.nonDerivation: This attribute defined by pkgs/by-name/no/nonDerivation/package.nix is not a derivation
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 00000000000..bd68dba1ded
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ }: null
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/expected b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/expected
new file mode 100644
index 00000000000..e6c92379010
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/expected
@@ -0,0 +1 @@
+pkgs.nonDerivation: This attribute defined by pkgs/by-name/no/nonDerivation/package.nix is not a derivation
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 00000000000..b021e28c214
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ }: { }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/one-letter/pkgs/by-name/a/a/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/pkgs/by-name/a/a/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/pkgs/by-name/a/a/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/all-packages.nix
new file mode 100644
index 00000000000..8bedb90d89a
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  nonDerivation = self.callPackage ./someDrv.nix { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected
new file mode 100644
index 00000000000..1c6377d8aef
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/expected
@@ -0,0 +1 @@
+pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 00000000000..bd68dba1ded
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ }: null
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/someDrv.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/someDrv.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/someDrv.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix
new file mode 100644
index 00000000000..4fad280ae1c
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  nonDerivation = null;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected
new file mode 100644
index 00000000000..1c6377d8aef
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/expected
@@ -0,0 +1 @@
+pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 00000000000..bd68dba1ded
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ }: null
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix
new file mode 100644
index 00000000000..4c521d2d446
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  nonDerivation = self.callPackage ({ }: { }) { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected
new file mode 100644
index 00000000000..1c6377d8aef
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/expected
@@ -0,0 +1 @@
+pkgs.nonDerivation: This attribute is not defined as `pkgs.callPackage pkgs/by-name/no/nonDerivation/package.nix { ... }`.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/pkgs/by-name/no/nonDerivation/package.nix
new file mode 100644
index 00000000000..bd68dba1ded
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/pkgs/by-name/no/nonDerivation/package.nix
@@ -0,0 +1 @@
+{ }: null
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/expected
new file mode 100644
index 00000000000..3ad4b8f820f
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/foo: This path is a file, but it should be a directory.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/pkgs/by-name/fo/foo b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/pkgs/by-name/fo/foo
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/pkgs/by-name/fo/foo
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/expected b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/expected
new file mode 100644
index 00000000000..67a0c69fe29
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/foo: "package.nix" must be a file.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/pkgs/by-name/fo/foo/package.nix/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/pkgs/by-name/fo/foo/package.nix/default.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/pkgs/by-name/fo/foo/package.nix/default.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/expected b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/expected
new file mode 100644
index 00000000000..7d20c32aad6
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/aa: File package.nix at line 2 contains the path expression "/foo" which cannot be resolved: No such file or directory (os error 2).
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..7a51ba1ec71
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1,3 @@
+{ someDrv }: someDrv // {
+  escape = /foo;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/expected b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/expected
new file mode 100644
index 00000000000..3d7fb64e80a
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/aa: File package.nix at line 2 contains the path expression "../." which may point outside the directory of that package.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..5989f52eb89
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1,3 @@
+{ someDrv }: someDrv // {
+  escape = ../.;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/expected b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/expected
new file mode 100644
index 00000000000..b0cdff4a477
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/aa: File package.nix at line 2 contains the nix search path expression "<nixpkgs>" which may point outside the directory of that package.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..864fdce1331
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1,3 @@
+{ someDrv }: someDrv // {
+  nixPath = <nixpkgs>;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/expected b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/expected
new file mode 100644
index 00000000000..281aba00923
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/aa: File invalid.nix could not be parsed by rnix: unexpected token at 28..29
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/invalid.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/invalid.nix
new file mode 100644
index 00000000000..ee6b002a529
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/invalid.nix
@@ -0,0 +1 @@
+this is not a valid nix file!
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/expected b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/expected
new file mode 100644
index 00000000000..ad662af27a8
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/expected
@@ -0,0 +1 @@
+pkgs/by-name/aa/aa: File package.nix at line 2 contains the path expression "./${"test"}", which is not yet supported and may point outside the directory of that package.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..a94ba754126
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1,3 @@
+{ someDrv }: someDrv // {
+  pathWithSubexpr = ./${"test"};
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/dir/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/dir/default.nix
new file mode 100644
index 00000000000..7e4a7548fec
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/dir/default.nix
@@ -0,0 +1,2 @@
+# Recursive
+../package.nix
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file.nix
new file mode 100644
index 00000000000..bd55e601bf6
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/file.nix
@@ -0,0 +1,2 @@
+# Recursive test
+import ./file.nix
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/package.nix
new file mode 100644
index 00000000000..474db5b0ebf
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/pkgs/by-name/aa/aa/package.nix
@@ -0,0 +1,5 @@
+{ someDrv }: someDrv // {
+  nixFile = ./file.nix;
+  nonNixFile = ./file;
+  directory = ./dir;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/shard-file/expected b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/expected
new file mode 100644
index 00000000000..447b38e6b6c
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo: This is a file, but it should be a directory.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/shard-file/pkgs/by-name/fo b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/pkgs/by-name/fo
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/pkgs/by-name/fo
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/success/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/success/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/success/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/expected b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/expected
new file mode 100644
index 00000000000..335c5d6b6e5
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/foo: Path package.nix is a symlink pointing to a path outside the directory of that package.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/pkgs/by-name/fo/foo/package.nix
new file mode 120000
index 00000000000..f079163d158
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+../../../../someDrv.nix
\ No newline at end of file
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/someDrv.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/someDrv.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/someDrv.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/expected b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/expected
new file mode 100644
index 00000000000..f622f3e7fd6
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/expected
@@ -0,0 +1 @@
+pkgs/by-name/fo/foo: Path foo.nix is a symlink which cannot be resolved: No such file or directory (os error 2).
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix
new file mode 120000
index 00000000000..49cd425a8cd
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix
@@ -0,0 +1 @@
+none.nix
\ No newline at end of file
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/someDrv.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/someDrv.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/someDrv.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/uppercase/pkgs/by-name/fo/FOO/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/pkgs/by-name/fo/FOO/package.nix
new file mode 100644
index 00000000000..a1b92efbbad
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/pkgs/by-name/fo/FOO/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix
new file mode 100644
index 00000000000..af25d145012
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/with-readme/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/pkgs/by-name/README.md
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/pkgs/by-name/README.md
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-05-27 10:28:16 +0000