From 1f8b5c039e9726ab4d17181a9eb9f63871007246 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 8 Jul 2023 18:51:49 +0200 Subject: lib.fix: Improve doc The original doc did not help with understanding at all, and the wikipedia link was actively harmful. --- lib/fixed-points.nix | 58 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix index a63f349b713..d1cb2dc030d 100644 --- a/lib/fixed-points.nix +++ b/lib/fixed-points.nix @@ -1,26 +1,66 @@ { lib, ... }: rec { /* - Compute the fixed point of the given function `f`, which is usually an - attribute set that expects its final, non-recursive representation as an - argument: + `fix f` computes the fixed point of the given function `f`. In other words, the return value is `x` in `x = f x`. + `f` is usually returns an attribute set that expects its final, non-recursive representation as an argument. + `f` must be a lazy function. + + **How it works** + + For context, Nix lets you define attribute set values in terms of other attributes using the `rec { }` attribute set literal syntax. + + ```nix + nix-repl> rec { + foo = "foo"; + bar = "bar"; + foobar = foo + bar; + } + { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - f = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } + + This is convenient when constructing a value to pass to a function for example, but a similar effect can be achieved with a `let` binding: + + ```nix + nix-repl> let self = { + foo = "foo"; + bar = "bar"; + foobar = self.foo + self.bar; + }; in self + { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - Nix evaluates this recursion until all references to `self` have been - resolved. At that point, the final result is returned and `f x = x` holds: + `let` bindings are nice, but as it is with `let` bindings in general, we may get more reuse out of the code by defining a function. + + ```nix + nix-repl> f = self: { + foo = "foo"; + bar = "bar"; + foobar = self.foo + self.bar; + } + ``` + + This is where `fix` comes in. Note that the body of the `fix` function + looks a lot like our earlier `let` binding, and that's no coincidence. + Fix is no more than such a recursive `let` binding, but with everything + except the recursion factored out into a function parameter `f`. + + ```nix + fix = f: + let self = f self; in self; + ``` + + So applying `fix` is another way to express our earlier examples. ``` nix-repl> fix f { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - Type: fix :: (a -> a) -> a + This example did not _need_ `fix`, and arguably it shouldn't be used in such an example. + However, `fix` is useful when your `f` is a parameter, or when it is constructed from higher order functions. - See https://en.wikipedia.org/wiki/Fixed-point_combinator for further - details. + Type: fix :: (a -> a) -> a */ fix = f: let x = f x; in x; -- cgit 1.4.1 From a0c77aecaaf24a32600b88d05406dfd57e1b8639 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Mon, 14 Aug 2023 01:32:29 -0700 Subject: lib.systems: add ubootArch u-boot has its own rosetta stone, almost but not exactly the same as the Linux kernel's. This commit adds it and the two cases where it diverges. --- lib/systems/default.nix | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/systems/default.nix b/lib/systems/default.nix index 40a2c88f32b..09024d587c5 100644 --- a/lib/systems/default.nix +++ b/lib/systems/default.nix @@ -178,6 +178,12 @@ rec { else if final.isLoongArch64 then "loongarch" else final.parsed.cpu.name; + # https://source.denx.de/u-boot/u-boot/-/blob/9bfb567e5f1bfe7de8eb41f8c6d00f49d2b9a426/common/image.c#L81-106 + ubootArch = + if final.isx86_32 then "x86" # not i386 + else if final.isMips64 then "mips64" # uboot *does* distinguish between mips32/mips64 + else final.linuxArch; # other cases appear to agree with linuxArch + qemuArch = if final.isAarch32 then "arm" else if final.isS390 && !final.isS390x then null -- cgit 1.4.1 From e39d57991886bcae28a1c7f51afa66ebdfb56a72 Mon Sep 17 00:00:00 2001 From: Thomas Gerbet Date: Sun, 27 Aug 2023 18:29:26 +0200 Subject: lib/licenses: add Elastic License 2.0, drop Elastic License This license was introduced in the Elastic search repository by this commit: https://github.com/elastic/elasticsearch/commit/a92a647b9f17d1bddf5c707490a19482c273eda3 It appears that all the the packages in nixpkgs using the Elastic License are using the v2.0. --- lib/licenses.nix | 6 +++--- pkgs/development/tools/bearer/default.nix | 2 +- pkgs/development/tools/misc/kibana/7.x.nix | 2 +- pkgs/servers/http/router/default.nix | 2 +- pkgs/servers/search/elasticsearch/7.x.nix | 2 +- pkgs/tools/admin/infra/default.nix | 2 +- pkgs/tools/misc/logstash/7.x.nix | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index 599e8ee53c9..6bbfb348f0b 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -413,9 +413,9 @@ in mkLicense lset) ({ fullName = "Eiffel Forum License v2.0"; }; - elastic = { - fullName = "ELASTIC LICENSE"; - url = "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt"; + elastic20 = { + fullName = "Elastic License 2.0"; + url = "https://github.com/elastic/elasticsearch/blob/main/licenses/ELASTIC-LICENSE-2.0.txt"; free = false; }; diff --git a/pkgs/development/tools/bearer/default.nix b/pkgs/development/tools/bearer/default.nix index d662ee9e42c..93e057112c1 100644 --- a/pkgs/development/tools/bearer/default.nix +++ b/pkgs/development/tools/bearer/default.nix @@ -39,7 +39,7 @@ buildGoModule rec { description = "Code security scanning tool (SAST) to discover, filter and prioritize security and privacy risks"; homepage = "https://github.com/bearer/bearer"; changelog = "https://github.com/Bearer/bearer/releases/tag/v${version}"; - license = with licenses; [ elastic ]; + license = with licenses; [ elastic20 ]; maintainers = with maintainers; [ fab ]; }; } diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix index eaf5247a416..fbed2b15488 100644 --- a/pkgs/development/tools/misc/kibana/7.x.nix +++ b/pkgs/development/tools/misc/kibana/7.x.nix @@ -53,7 +53,7 @@ in stdenv.mkDerivation rec { meta = with lib; { description = "Visualize logs and time-stamped data"; homepage = "http://www.elasticsearch.org/overview/kibana"; - license = licenses.elastic; + license = licenses.elastic20; maintainers = with maintainers; [ offline basvandijk ]; platforms = with platforms; unix; }; diff --git a/pkgs/servers/http/router/default.nix b/pkgs/servers/http/router/default.nix index 31fc8332c96..b7b33c1d176 100644 --- a/pkgs/servers/http/router/default.nix +++ b/pkgs/servers/http/router/default.nix @@ -40,7 +40,7 @@ rustPlatform.buildRustPackage rec { meta = with lib; { description = "A configurable, high-performance routing runtime for Apollo Federation"; homepage = "https://www.apollographql.com/docs/router/"; - license = licenses.elastic; + license = licenses.elastic20; maintainers = [ maintainers.bbigras ]; }; } diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix index 6eea9d751d5..1fbd64df29c 100644 --- a/pkgs/servers/search/elasticsearch/7.x.nix +++ b/pkgs/servers/search/elasticsearch/7.x.nix @@ -76,7 +76,7 @@ stdenv.mkDerivation rec { binaryBytecode binaryNativeCode ]; - license = licenses.elastic; + license = licenses.elastic20; platforms = platforms.unix; maintainers = with maintainers; [ apeschar basvandijk ]; }; diff --git a/pkgs/tools/admin/infra/default.nix b/pkgs/tools/admin/infra/default.nix index f5189203943..bfb44acc6ed 100644 --- a/pkgs/tools/admin/infra/default.nix +++ b/pkgs/tools/admin/infra/default.nix @@ -21,7 +21,7 @@ buildGoModule rec { description = "Infra manages access to infrastructure such as Kubernetes"; homepage = "https://github.com/infrahq/infra"; changelog = "https://github.com/infrahq/infra/raw/v${version}/CHANGELOG.md"; - license = licenses.elastic; + license = licenses.elastic20; maintainers = with maintainers; [ peterromfeldhk ]; }; } diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix index 7bbc5bd6a0d..2ca8f925827 100644 --- a/pkgs/tools/misc/logstash/7.x.nix +++ b/pkgs/tools/misc/logstash/7.x.nix @@ -72,7 +72,7 @@ let binaryBytecode # source bundles dependencies as jars binaryNativeCode # bundled jruby includes native code ]; - license = if enableUnfree then licenses.elastic else licenses.asl20; + license = if enableUnfree then licenses.elastic20 else licenses.asl20; platforms = platforms.unix; maintainers = with maintainers; [ wjlroe offline basvandijk ]; }; -- cgit 1.4.1 From 20b83a46d62422b7070035a32ae23cce408d4034 Mon Sep 17 00:00:00 2001 From: Ratchanan Srirattanamet Date: Mon, 25 Sep 2023 04:19:12 +0700 Subject: netdata: disable installation of non-free v2 dashboard The v2 dashboard (included since version 1.41.0) is licensed under Netdata Cloud UI License v1.0, which is a non-free license. Patch the source code so that the v2 is not included in the source and the result derivation. Users that want the v2 dashboard can still request it by overriding this derivation and pass `withCloudUi = true`. Co-authored-by: Raito Bezarius Fixes: https://github.com/NixOS/nixpkgs/issues/256964 --- lib/licenses.nix | 8 ++++++++ pkgs/tools/system/netdata/default.nix | 25 ++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index 0fd64108537..dcc16edd90d 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -840,6 +840,14 @@ in mkLicense lset) ({ fullName = "University of Illinois/NCSA Open Source License"; }; + ncul1 = { + spdxId = "NCUL1"; + fullName = "Netdata Cloud UI License v1.0"; + free = false; + redistributable = true; # Only if used in Netdata products. + url = "https://raw.githubusercontent.com/netdata/netdata/master/web/gui/v2/LICENSE.md"; + }; + nlpl = { spdxId = "NLPL"; fullName = "No Limit Public License"; diff --git a/pkgs/tools/system/netdata/default.nix b/pkgs/tools/system/netdata/default.nix index e3dbabe1f52..f380bf4fd6f 100644 --- a/pkgs/tools/system/netdata/default.nix +++ b/pkgs/tools/system/netdata/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, autoreconfHook, pkg-config, makeWrapper +{ lib, stdenv, fetchFromGitHub, fetchpatch, autoreconfHook, pkg-config, makeWrapper , CoreFoundation, IOKit, libossp_uuid , nixosTests , netdata-go-plugins @@ -9,6 +9,7 @@ , withIpmi ? (!stdenv.isDarwin), freeipmi , withNetfilter ? (!stdenv.isDarwin), libmnl, libnetfilter_acct , withCloud ? (!stdenv.isDarwin), json_c +, withCloudUi ? false , withConnPubSub ? false, google-cloud-cpp, grpc , withConnPrometheus ? false, snappy , withSsl ? true, openssl @@ -24,8 +25,17 @@ stdenv.mkDerivation rec { owner = "netdata"; repo = "netdata"; rev = "v${version}"; - hash = "sha256-8L8PhPgNIHvw+Dcx2D6OE8fp2+GEYOc9wEIoPJSqXME="; + hash = if withCloudUi + then "sha256-8L8PhPgNIHvw+Dcx2D6OE8fp2+GEYOc9wEIoPJSqXME=" + else "sha256-J/pKKxTNoSwvsyVaRsnazQQqu2C8zx1QEAkB+gkR5lU="; fetchSubmodules = true; + + # Remove v2 dashboard distributed under NCUL1. Make sure an empty + # Makefile.am exists, as autoreconf will get confused otherwise. + postFetch = lib.optionalString (!withCloudUi) '' + rm -rf $out/web/gui/v2/* + touch $out/web/gui/v2/Makefile.am + ''; }; strictDeps = true; @@ -53,6 +63,12 @@ stdenv.mkDerivation rec { # Avoid build-only inputs in closure leaked by configure command: # https://github.com/NixOS/nixpkgs/issues/175693#issuecomment-1143344162 ./skip-CONFIGURE_COMMAND.patch + + # Allow building without non-free v2 dashboard. + (fetchpatch { + url = "https://github.com/peat-psuwit/netdata/commit/6ccbdd1500db2b205923968688d5f1777430a326.patch"; + hash = "sha256-jAyk5HlxdjFn5IP6jOKP8/SXOraMQSA6r1krThe+s7g="; + }) ]; # Guard against unused buld-time development inputs in closure. Without @@ -98,6 +114,8 @@ stdenv.mkDerivation rec { "--disable-dbengine" ] ++ lib.optionals (!withCloud) [ "--disable-cloud" + ] ++ lib.optionals (!withCloudUi) [ + "--disable-cloud-ui" ]; postFixup = '' @@ -118,7 +136,8 @@ stdenv.mkDerivation rec { description = "Real-time performance monitoring tool"; homepage = "https://www.netdata.cloud/"; changelog = "https://github.com/netdata/netdata/releases/tag/v${version}"; - license = licenses.gpl3Plus; + license = [ licenses.gpl3Plus ] + ++ lib.optionals (withCloudUi) [ licenses.ncul1 ]; platforms = platforms.unix; maintainers = with maintainers; [ raitobezarius ]; }; -- cgit 1.4.1 From 5d5c872bdffb0542662516aec96ae05d3762fbe1 Mon Sep 17 00:00:00 2001 From: FavoritoHJS <69703458+FavoritoHJS@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:00:44 +0000 Subject: licenses: add Fraunhofer FDK AAC Codec Library This license is used by the Fraunhofer FDK codec, which is currently mislicensed to Apache 2.0. Of course, this can't be corrected without having the proper license available in lib.licenses can't it?? Thanks @trofi for noticing this has an SPDX ID. --- lib/licenses.nix | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index 0fd64108537..615a6b38815 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -481,6 +481,11 @@ in mkLicense lset) ({ free = false; }; + fraunhofer-fdk = { + fullName = "Fraunhofer FDK AAC Codec Library"; + spdxId = "FDK-AAC"; + }; + free = { fullName = "Unspecified free software license"; }; -- cgit 1.4.1 From 7a0b5c36c943f4f257c93f3c3e7fed22ffe2d67c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 19:21:02 +0200 Subject: lib.lists.foldl': Remove fallback Nix 2.3, the minimum Nix version supported by Nixpkgs, has `builtins.foldl'` already. --- lib/lists.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/lists.nix b/lib/lists.nix index 0800aeb6545..4a13550ca9e 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -94,7 +94,7 @@ rec { Type: foldl' :: (b -> a -> b) -> b -> [a] -> b */ - foldl' = builtins.foldl' or foldl; + foldl' = builtins.foldl'; /* Map with index starting from 0 -- cgit 1.4.1 From 9893fee947ceba487f0475bbecfea6d5959e2e6f Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 22 Sep 2023 02:38:15 +0200 Subject: lib.lists.foldl': Add tests --- lib/tests/misc.nix | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'lib') diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 80223dccb26..50d615c5be3 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -505,6 +505,38 @@ runTests { }; }; + testFoldl'Empty = { + expr = foldl' (acc: el: abort "operation not called") 0 [ ]; + expected = 0; + }; + + testFoldl'IntegerAdding = { + expr = foldl' (acc: el: acc + el) 0 [ 1 2 3 ]; + expected = 6; + }; + + # The accumulator isn't forced deeply + testFoldl'NonDeep = { + expr = take 3 (foldl' + (acc: el: [ el ] ++ acc) + [ (abort "unevaluated list entry") ] + [ 1 2 3 ]); + expected = [ 3 2 1 ]; + }; + + # The same as builtins.foldl', lib.foldl' doesn't evaluate the first accumulator strictly + testFoldl'StrictInitial = { + expr = (builtins.tryEval (foldl' (acc: el: el) (throw "hello") [])).success; + expected = true; + }; + + # Make sure we don't get a stack overflow for large lists + # This number of elements would notably cause a stack overflow if it was implemented without the `foldl'` builtin + testFoldl'Large = { + expr = foldl' (acc: el: acc + el) 0 (range 0 100000); + expected = 5000050000; + }; + testTake = testAllTrue [ ([] == (take 0 [ 1 2 3 ])) ([1] == (take 1 [ 1 2 3 ])) -- cgit 1.4.1 From 857a844ea8f1736e42f9c14c992d95be7b83a7c4 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 22 Sep 2023 02:42:29 +0200 Subject: lib.lists.foldl': Redo documentation Co-Authored-By: Robert Hensing --- lib/lists.nix | 55 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/lists.nix b/lib/lists.nix index 4a13550ca9e..8c5099084bb 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -86,15 +86,56 @@ rec { else op (foldl' (n - 1)) (elemAt list n); in foldl' (length list - 1); - /* Strict version of `foldl`. + /* + Reduce a list by applying a binary operator from left to right, + starting with an initial accumulator. - The difference is that evaluation is forced upon access. Usually used - with small whole results (in contrast with lazily-generated list or large - lists where only a part is consumed.) + After each application of the operator, the resulting value is evaluated. + This behavior makes this function stricter than [`foldl`](#function-library-lib.lists.foldl). - Type: foldl' :: (b -> a -> b) -> b -> [a] -> b - */ - foldl' = builtins.foldl'; + A call like + + ```nix + foldl' op acc₀ [ x₀ x₁ x₂ ... xₙ₋₁ xₙ ] + ``` + + is (denotationally) equivalent to the following, + but with the added benefit that `foldl'` itself will never overflow the stack. + + ```nix + let + acc₁ = op acc₀ x₀ ; + acc₂ = builtins.seq acc₁ (op acc₁ x₁ ); + acc₃ = builtins.seq acc₂ (op acc₂ x₂ ); + ... + accₙ = builtins.seq accₙ₋₁ (op accₙ₋₁ xₙ₋₁); + accₙ₊₁ = builtins.seq accₙ (op accₙ xₙ ); + in + accₙ₊₁ + + # Or ignoring builtins.seq + op (op (... (op (op (op acc₀ x₀) x₁) x₂) ...) xₙ₋₁) xₙ + ``` + + Type: foldl' :: (acc -> x -> acc) -> acc -> [x] -> acc + + Example: + foldl' (acc: x: acc + x) 0 [1 2 3] + => 6 + */ + foldl' = + /* The binary operation to run, where the two arguments are: + + 1. `acc`: The current accumulator value: Either the initial one for the first iteration, or the result of the previous iteration + 2. `x`: The corresponding list element for this iteration + */ + op: + # The initial accumulator value + acc: + # The list to fold + list: + + builtins.foldl' op acc list; /* Map with index starting from 0 -- cgit 1.4.1 From 3b6169f87be45c77ec4b56d118a5e2c718ff3f2b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 22 Sep 2023 12:22:04 +0200 Subject: lib.lists.foldl': Make strict in the initial accumulator To maintain backwards compatibility, this can't be changed in the Nix language. We can however ensure that the version Nixpkgs has the more intuitive behavior. --- lib/attrsets.nix | 2 +- lib/lists.nix | 13 ++++++++++--- lib/tests/misc.nix | 4 ++-- nixos/doc/manual/release-notes/rl-2311.section.md | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/attrsets.nix b/lib/attrsets.nix index 77e36d3271f..11932c05dd2 100644 --- a/lib/attrsets.nix +++ b/lib/attrsets.nix @@ -392,7 +392,7 @@ rec { foldlAttrs :: ( a -> String -> b -> a ) -> a -> { ... :: b } -> a */ foldlAttrs = f: init: set: - foldl' + builtins.foldl' (acc: name: f acc name set.${name}) init (attrNames set); diff --git a/lib/lists.nix b/lib/lists.nix index 8c5099084bb..3835e3ba69c 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -90,9 +90,12 @@ rec { Reduce a list by applying a binary operator from left to right, starting with an initial accumulator. - After each application of the operator, the resulting value is evaluated. + Before each application of the operator, the accumulator value is evaluated. This behavior makes this function stricter than [`foldl`](#function-library-lib.lists.foldl). + Unlike [`builtins.foldl'`](https://nixos.org/manual/nix/unstable/language/builtins.html#builtins-foldl'), + the initial accumulator argument is evaluated before the first iteration. + A call like ```nix @@ -104,7 +107,7 @@ rec { ```nix let - acc₁ = op acc₀ x₀ ; + acc₁ = builtins.seq acc₀ (op acc₀ x₀ ); acc₂ = builtins.seq acc₁ (op acc₁ x₁ ); acc₃ = builtins.seq acc₂ (op acc₂ x₂ ); ... @@ -135,7 +138,11 @@ rec { # The list to fold list: - builtins.foldl' op acc list; + # The builtin `foldl'` is a bit lazier than one might expect. + # See https://github.com/NixOS/nix/pull/7158. + # In particular, the initial accumulator value is not forced before the first iteration starts. + builtins.seq acc + (builtins.foldl' op acc list); /* Map with index starting from 0 diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 50d615c5be3..d40d9204988 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -524,10 +524,10 @@ runTests { expected = [ 3 2 1 ]; }; - # The same as builtins.foldl', lib.foldl' doesn't evaluate the first accumulator strictly + # Compared to builtins.foldl', lib.foldl' evaluates the first accumulator strictly too testFoldl'StrictInitial = { expr = (builtins.tryEval (foldl' (acc: el: el) (throw "hello") [])).success; - expected = true; + expected = false; }; # Make sure we don't get a stack overflow for large lists diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index cdb73fb49fa..011fa84c96a 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -226,6 +226,9 @@ - `networking.networkmanager.firewallBackend` was removed as NixOS is now using iptables-nftables-compat even when using iptables, therefore Networkmanager now uses the nftables backend unconditionally. +- [`lib.lists.foldl'`](https://nixos.org/manual/nixpkgs/stable#function-library-lib.lists.foldl-prime) now always evaluates the initial accumulator argument first. + If you depend on the lazier behavior, consider using [`lib.lists.foldl`](https://nixos.org/manual/nixpkgs/stable#function-library-lib.lists.foldl) or [`builtins.foldl'`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-foldl') instead. + - `rome` was removed because it is no longer maintained and is succeeded by `biome`. - The `services.mtr-exporter.target` has been removed in favor of `services.mtr-exporter.jobs` which allows specifying multiple targets. -- cgit 1.4.1 From dd72ff27f783ff62c93d78f625633a09c4658344 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 22 Sep 2023 12:24:27 +0200 Subject: lib.attrsets.foldlAttrs: Make stricter See the parent commit for the same change to lib.lists.foldl' --- lib/attrsets.nix | 10 +++++----- lib/tests/misc.nix | 4 ++-- nixos/doc/manual/release-notes/rl-2311.section.md | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/attrsets.nix b/lib/attrsets.nix index 11932c05dd2..b8960cf73f2 100644 --- a/lib/attrsets.nix +++ b/lib/attrsets.nix @@ -338,7 +338,7 @@ rec { ); /* - Like builtins.foldl' but for attribute sets. + Like [`lib.lists.foldl'`](#function-library-lib.lists.foldl-prime) but for attribute sets. Iterates over every name-value pair in the given attribute set. The result of the callback function is often called `acc` for accumulator. It is passed between callbacks from left to right and the final `acc` is the return value of `foldlAttrs`. @@ -372,9 +372,9 @@ rec { 123 foldlAttrs - (_: _: v: v) - (throw "initial accumulator not needed") - { z = 3; a = 2; }; + (acc: _: _: acc) + 3 + { z = throw "value not needed"; a = throw "value not needed"; }; -> 3 @@ -392,7 +392,7 @@ rec { foldlAttrs :: ( a -> String -> b -> a ) -> a -> { ... :: b } -> a */ foldlAttrs = f: init: set: - builtins.foldl' + foldl' (acc: name: f acc name set.${name}) init (attrNames set); diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index d40d9204988..ec306acbb76 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -740,7 +740,7 @@ runTests { # should just return the initial value emptySet = foldlAttrs (throw "function not needed") 123 { }; # should just evaluate to the last value - accNotNeeded = foldlAttrs (_acc: _name: v: v) (throw "accumulator not needed") { z = 3; a = 2; }; + valuesNotNeeded = foldlAttrs (acc: _name: _v: acc) 3 { z = throw "value z not needed"; a = throw "value a not needed"; }; # the accumulator doesnt have to be an attrset it can be as trivial as being just a number or string trivialAcc = foldlAttrs (acc: _name: v: acc * 10 + v) 1 { z = 1; a = 2; }; }; @@ -750,7 +750,7 @@ runTests { names = [ "bar" "foo" ]; }; emptySet = 123; - accNotNeeded = 3; + valuesNotNeeded = 3; trivialAcc = 121; }; }; diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 011fa84c96a..b5a1986b7a3 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -229,6 +229,8 @@ - [`lib.lists.foldl'`](https://nixos.org/manual/nixpkgs/stable#function-library-lib.lists.foldl-prime) now always evaluates the initial accumulator argument first. If you depend on the lazier behavior, consider using [`lib.lists.foldl`](https://nixos.org/manual/nixpkgs/stable#function-library-lib.lists.foldl) or [`builtins.foldl'`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-foldl') instead. +- [`lib.attrsets.foldlAttrs`](https://nixos.org/manual/nixpkgs/stable#function-library-lib.attrsets.foldlAttrs) now always evaluates the initial accumulator argument first. + - `rome` was removed because it is no longer maintained and is succeeded by `biome`. - The `services.mtr-exporter.target` has been removed in favor of `services.mtr-exporter.jobs` which allows specifying multiple targets. -- cgit 1.4.1 From cc3383d12b25f8e1f2b23fdb098a9cb1b22dfa58 Mon Sep 17 00:00:00 2001 From: Anderson Torres Date: Wed, 20 Sep 2023 20:41:57 -0300 Subject: lib: add cmakeOptionType, cmakeBool and cmakeFeature --- lib/default.nix | 1 + lib/strings.nix | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) (limited to 'lib') diff --git a/lib/default.nix b/lib/default.nix index e4bf45aac3b..169f013191b 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -106,6 +106,7 @@ let upperChars toLower toUpper addContextFrom splitString removePrefix removeSuffix versionOlder versionAtLeast getName getVersion + cmakeOptionType cmakeBool cmakeFeature mesonOption mesonBool mesonEnable nameFromURL enableFeature enableFeatureAs withFeature withFeatureAs fixedWidthString fixedWidthNumber diff --git a/lib/strings.nix b/lib/strings.nix index df891c89988..d7642ce10fa 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -741,6 +741,64 @@ rec { name = head (splitString sep filename); in assert name != filename; name; + /* Create a "-D:=" string that can be passed to typical + CMake invocations. + + Type: cmakeOptionType :: string -> string -> string -> string + + @param feature The feature to be set + @param type The type of the feature to be set, as described in + https://cmake.org/cmake/help/latest/command/set.html + the possible values (case insensitive) are: + BOOL FILEPATH PATH STRING INTERNAL + @param value The desired value + + Example: + cmakeOptionType "string" "ENGINE" "sdl2" + => "-DENGINE:STRING=sdl2" + */ + cmakeOptionType = type: feature: value: + assert (lib.elem (lib.toUpper type) + [ "BOOL" "FILEPATH" "PATH" "STRING" "INTERNAL" ]); + assert (lib.isString feature); + assert (lib.isString value); + "-D${feature}:${lib.toUpper type}=${value}"; + + /* Create a -D={TRUE,FALSE} string that can be passed to typical + CMake invocations. + + Type: cmakeBool :: string -> bool -> string + + @param condition The condition to be made true or false + @param flag The controlling flag of the condition + + Example: + cmakeBool "ENABLE_STATIC_LIBS" false + => "-DENABLESTATIC_LIBS:BOOL=FALSE" + */ + cmakeBool = condition: flag: + assert (lib.isString condition); + assert (lib.isBool flag); + cmakeOptionType "bool" condition (lib.toUpper (lib.boolToString flag)); + + /* Create a -D:STRING= string that can be passed to typical + CMake invocations. + This is the most typical usage, so it deserves a special case. + + Type: cmakeFeature :: string -> string -> string + + @param condition The condition to be made true or false + @param flag The controlling flag of the condition + + Example: + cmakeFeature "MODULES" "badblock" + => "-DMODULES:STRING=badblock" + */ + cmakeFeature = feature: value: + assert (lib.isString feature); + assert (lib.isString value); + cmakeOptionType "string" feature value; + /* Create a -D= string that can be passed to typical Meson invocations. -- cgit 1.4.1 From a50cea84be6f4706e563b6785418b85977b79a0c Mon Sep 17 00:00:00 2001 From: linsui Date: Sat, 23 Sep 2023 02:40:03 +0800 Subject: lib.gvariant: make the tests really work --- lib/tests/modules.sh | 3 + lib/tests/modules/gvariant.nix | 142 ++++++++++++++++------------------------- 2 files changed, 58 insertions(+), 87 deletions(-) (limited to 'lib') diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 93fb1df9332..05c99e6de83 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -91,6 +91,9 @@ checkConfigOutput '^true$' config.result ./test-mergeAttrDefinitionsWithPrio.nix # is the option. checkConfigOutput '^true$' config.result ./module-argument-default.nix +# gvariant +checkConfigOutput '^true$' config.assertion ./gvariant.nix + # types.pathInStore checkConfigOutput '".*/store/0lz9p8xhf89kb1c1kk6jxrzskaiygnlh-bash-5.2-p15.drv"' config.pathInStore.ok1 ./types.nix checkConfigOutput '".*/store/0fb3ykw9r5hpayd05sr0cizwadzq1d8q-bash-5.2-p15"' config.pathInStore.ok2 ./types.nix diff --git a/lib/tests/modules/gvariant.nix b/lib/tests/modules/gvariant.nix index a792ebf85b7..ba452c0287a 100644 --- a/lib/tests/modules/gvariant.nix +++ b/lib/tests/modules/gvariant.nix @@ -1,93 +1,61 @@ { config, lib, ... }: -let inherit (lib) concatStringsSep mapAttrsToList mkMerge mkOption types gvariant; -in { - options.examples = mkOption { type = types.attrsOf gvariant; }; +{ + options = { + examples = lib.mkOption { type = lib.types.attrs; }; + assertion = lib.mkOption { type = lib.types.bool; }; + }; config = { - examples = with gvariant; - mkMerge [ - { bool = true; } - { bool = true; } - - { float = 3.14; } - - { int32 = mkInt32 (- 42); } - { int32 = mkInt32 (- 42); } - - { uint32 = mkUint32 42; } - { uint32 = mkUint32 42; } - - { int16 = mkInt16 (-42); } - { int16 = mkInt16 (-42); } - - { uint16 = mkUint16 42; } - { uint16 = mkUint16 42; } - - { int64 = mkInt64 (-42); } - { int64 = mkInt64 (-42); } - - { uint64 = mkUint64 42; } - { uint64 = mkUint64 42; } - - { array1 = [ "one" ]; } - { array1 = mkArray [ "two" ]; } - { array2 = mkArray [ (mkInt32 1) ]; } - { array2 = mkArray [ (nkUint32 2) ]; } - - { emptyArray1 = [ ]; } - { emptyArray2 = mkEmptyArray type.uint32; } - - { string = "foo"; } - { string = "foo"; } - { - escapedString = '' - '\ - ''; - } - - { tuple = mkTuple [ (mkInt32 1) [ "foo" ] ]; } - - { maybe1 = mkNothing type.string; } - { maybe2 = mkJust (mkUint32 4); } - - { variant1 = mkVariant "foo"; } - { variant2 = mkVariant 42; } - - { dictionaryEntry = mkDictionaryEntry (mkInt32 1) [ "foo" ]; } - ]; - - assertions = [ - { - assertion = ( - let - mkLine = n: v: "${n} = ${toString (gvariant.mkValue v)}"; - result = concatStringsSep "\n" (mapAttrsToList mkLine config.examples); - in - result + "\n" - ) == '' - array1 = @as ['one','two'] - array2 = @au [1,2] - bool = true - dictionaryEntry = @{ias} {1,@as ['foo']} - emptyArray1 = @as [] - emptyArray2 = @au [] - escapedString = '\'\\\n' - float = 3.140000 - int = -42 - int16 = @n -42 - int64 = @x -42 - maybe1 = @ms nothing - maybe2 = just @u 4 - string = 'foo' - tuple = @(ias) (1,@as ['foo']) - uint16 = @q 42 - uint32 = @u 42 - uint64 = @t 42 - variant1 = @v <'foo'> - variant2 = @v <42> - ''; - } - ]; + examples = with lib.gvariant; { + bool = true; + float = 3.14; + int32 = mkInt32 (- 42); + uint32 = mkUint32 42; + int16 = mkInt16 (-42); + uint16 = mkUint16 42; + int64 = mkInt64 (-42); + uint64 = mkUint64 42; + array1 = [ "one" ]; + array2 = mkArray [ (mkInt32 1) ]; + array3 = mkArray [ (mkUint32 2) ]; + emptyArray = mkEmptyArray type.uint32; + string = "foo"; + escapedString = '' + '\ + ''; + tuple = mkTuple [ (mkInt32 1) [ "foo" ] ]; + maybe1 = mkNothing type.string; + maybe2 = mkJust (mkUint32 4); + variant = mkVariant "foo"; + dictionaryEntry = mkDictionaryEntry (mkInt32 1) [ "foo" ]; + }; + + assertion = + let + mkLine = n: v: "${n} = ${toString (lib.gvariant.mkValue v)}"; + result = lib.concatStringsSep "\n" (lib.mapAttrsToList mkLine config.examples); + in + (result + "\n") == '' + array1 = @as ['one'] + array2 = @ai [1] + array3 = @au [@u 2] + bool = true + dictionaryEntry = @{ias} {1,@as ['foo']} + emptyArray = @au [] + escapedString = '\'\\\n' + float = 3.140000 + int16 = @n -42 + int32 = -42 + int64 = @x -42 + maybe1 = @ms nothing + maybe2 = just @u 4 + string = 'foo' + tuple = @(ias) (1,@as ['foo']) + uint16 = @q 42 + uint32 = @u 42 + uint64 = @t 42 + variant = <'foo'> + ''; }; } -- cgit 1.4.1 From 3cbbea53a12ebd461e047e594027ee6777f36e68 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 29 Sep 2023 20:07:08 +0200 Subject: lib.fileset: Don't use non-reproducible ulimit for stack overflow testing --- lib/fileset/tests.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 0ea96859e7a..774b8f29d93 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -467,12 +467,13 @@ for i in $(seq 1000); do tree[$i/a]=1 tree[$i/b]=0 done -( - # Locally limit the maximum stack size to 100 * 1024 bytes - # If unions was implemented recursively, this would stack overflow - ulimit -s 100 - checkFileset 'unions (mapAttrsToList (name: _: ./. + "/${name}/a") (builtins.readDir ./.))' -) +# This is actually really hard to test: +# A lot of files would be needed to cause a stack overflow. +# And while we could limit the maximum stack size using `ulimit -s`, +# that turns out to not be very deterministic: https://github.com/NixOS/nixpkgs/pull/256417#discussion_r1339396686. +# Meanwhile, the test infra here is not the fastest, creating 10000 would be too slow. +# So, just using 1000 files for now. +checkFileset 'unions (mapAttrsToList (name: _: ./. + "/${name}/a") (builtins.readDir ./.))' # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets -- cgit 1.4.1 From 4f35f003e6e5b800be75e3985054e5fce2dea50a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 26 Sep 2023 00:26:00 +0200 Subject: lib.fileset: Internal representation v3, empty value without a base `unions []` now works! Notably the new empty value without a base is not exposed in the interface. I don't know of any use case for it. --- lib/fileset/README.md | 29 +++++++++++++++++++- lib/fileset/default.nix | 7 ++--- lib/fileset/internal.nix | 69 +++++++++++++++++++++++++++++++++++++++--------- lib/fileset/tests.sh | 40 +++++++++++++++++++++++----- 4 files changed, 119 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 6e57f1f8f2b..2ef7b81e885 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -41,9 +41,16 @@ An attribute set with these values: - `_type` (constant string `"fileset"`): Tag to indicate this value is a file set. -- `_internalVersion` (constant `2`, the current version): +- `_internalVersion` (constant `3`, the current version): Version of the representation. +- `_internalIsEmptyWithoutBase` (bool): + Whether this file set is the empty file set without a base path. + If `true`, `_internalBase*` and `_internalTree` are not set. + This is the only way to represent an empty file set without needing a base path. + + Such a value can be used as the identity element for `union` and the return value of `unions []` and co. + - `_internalBase` (path): Any files outside of this path cannot influence the set of files. This is always a directory. @@ -111,6 +118,26 @@ Arguments: - (+) This can be removed later, if we discover it's too restrictive - (-) It leads to errors when a sensible result could sometimes be returned, such as in the above example. +### Empty file set without a base + +There is a special representation for an empty file set without a base path. +This is used for return values that should be empty but when there's no base path that would makes sense. + +Arguments: +- Alternative: This could also be represented using `_internalBase = /.` and `_internalTree = null`. + - (+) Removes the need for a special representation. + - (-) Due to [influence tracking](#influence-tracking), + `union empty ./.` would have `/.` as the base path, + which would then prevent `toSource { root = ./.; fileset = union empty ./.; }` from working, + which is not as one would expect. + - (-) With the assumption that there can be multiple filesystem roots (as established with the [path library](../path/README.md)), + this would have to cause an error with `union empty pathWithAnotherFilesystemRoot`, + which is not as one would expect. +- Alternative: Do not have such a value and error when it would be needed as a return value + - (+) Removes the need for a special representation. + - (-) Leaves us with no identity element for `union` and no reasonable return value for `unions []`. + From a set theory perspective, which has a well-known notion of empty sets, this is unintuitive. + ### Empty directories File sets can only represent a _set_ of local files, directories on their own are not representable. diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 88c8dcd1a70..ab26112c947 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -156,7 +156,7 @@ If a directory does not recursively contain any file, it is omitted from the sto lib.fileset.toSource: `root` is of type ${typeOf root}, but it should be a path instead.'' # Currently all Nix paths have the same filesystem root, but this could change in the future. # See also ../path/README.md - else if rootFilesystemRoot != filesetFilesystemRoot then + else if ! fileset._internalIsEmptyWithoutBase && rootFilesystemRoot != filesetFilesystemRoot then throw '' lib.fileset.toSource: Filesystem roots are not the same for `fileset` and `root` ("${toString root}"): `root`: root "${toString rootFilesystemRoot}" @@ -170,7 +170,7 @@ If a directory does not recursively contain any file, it is omitted from the sto lib.fileset.toSource: `root` (${toString root}) is a file, but it should be a directory instead. Potential solutions: - If you want to import the file into the store _without_ a containing directory, use string interpolation or `builtins.path` instead of this function. - If you want to import the file into the store _with_ a containing directory, set `root` to the containing directory, such as ${toString (dirOf root)}, and set `fileset` to the file path.'' - else if ! hasPrefix root fileset._internalBase then + else if ! fileset._internalIsEmptyWithoutBase && ! hasPrefix root fileset._internalBase then throw '' lib.fileset.toSource: `fileset` could contain files in ${toString fileset._internalBase}, which is not under the `root` (${toString root}). Potential solutions: - Set `root` to ${toString fileset._internalBase} or any directory higher up. This changes the layout of the resulting store path. @@ -264,9 +264,6 @@ If a directory does not recursively contain any file, it is omitted from the sto filesets: if ! isList filesets then throw "lib.fileset.unions: Expected argument to be a list, but got a ${typeOf filesets}." - else if filesets == [ ] then - # TODO: This could be supported, but requires an extra internal representation for the empty file set, which would be special for not having a base path. - throw "lib.fileset.unions: Expected argument to be a list with at least one element, but it contains no elements." else pipe filesets [ # Annotate the elements with context, used by _coerceMany for better errors diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 2c329edb390..b3dbc5b33dc 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -28,6 +28,7 @@ let drop elemAt filter + findFirst findFirstIndex foldl' head @@ -64,7 +65,7 @@ rec { # - Increment this version # - Add an additional migration function below # - Update the description of the internal representation in ./README.md - _currentVersion = 2; + _currentVersion = 3; # Migrations between versions. The 0th element converts from v0 to v1, and so on migrations = [ @@ -89,8 +90,34 @@ rec { _internalVersion = 2; } ) + + # Convert v2 into v3: filesetTree's now have a representation for an empty file set without a base path + ( + filesetV2: + filesetV2 // { + # All v1 file sets are not the new empty file set + _internalIsEmptyWithoutBase = false; + _internalVersion = 3; + } + ) ]; + _noEvalMessage = '' + lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.''; + + # The empty file set without a base path + _emptyWithoutBase = { + _type = "fileset"; + + _internalVersion = _currentVersion; + + # The one and only! + _internalIsEmptyWithoutBase = true; + + # Double __ to make it be evaluated and ordered first + __noEval = throw _noEvalMessage; + }; + # Create a fileset, see ./README.md#fileset # Type: path -> filesetTree -> fileset _create = base: tree: @@ -103,14 +130,15 @@ rec { _type = "fileset"; _internalVersion = _currentVersion; + + _internalIsEmptyWithoutBase = false; _internalBase = base; _internalBaseRoot = parts.root; _internalBaseComponents = components parts.subpath; _internalTree = tree; # Double __ to make it be evaluated and ordered first - __noEval = throw '' - lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.''; + __noEval = throw _noEvalMessage; }; # Coerce a value to a fileset, erroring when the value cannot be coerced. @@ -155,14 +183,20 @@ rec { _coerce "${functionContext}: ${context}" value ) list; - firstBaseRoot = (head filesets)._internalBaseRoot; + # Find the first value with a base, there may be none! + firstWithBase = findFirst (fileset: ! fileset._internalIsEmptyWithoutBase) null filesets; + # This value is only accessed if first != null + firstBaseRoot = firstWithBase._internalBaseRoot; # Finds the first element with a filesystem root different than the first element, if any differentIndex = findFirstIndex (fileset: - firstBaseRoot != fileset._internalBaseRoot + # The empty value without a base doesn't have a base path + ! fileset._internalIsEmptyWithoutBase + && firstBaseRoot != fileset._internalBaseRoot ) null filesets; in - if differentIndex != null then + # Only evaluates `differentIndex` if there are any elements with a base + if firstWithBase != null && differentIndex != null then throw '' ${functionContext}: Filesystem roots are not the same: ${(head list).context}: root "${toString firstBaseRoot}" @@ -311,7 +345,7 @@ rec { # Special case because the code below assumes that the _internalBase is always included in the result # which shouldn't be done when we have no files at all in the base # This also forces the tree before returning the filter, leads to earlier error messages - if tree == null then + if fileset._internalIsEmptyWithoutBase || tree == null then empty else nonEmpty; @@ -321,7 +355,12 @@ rec { # Type: [ Fileset ] -> Fileset _unionMany = filesets: let - first = head filesets; + # All filesets that have a base, aka not the ones that are the empty value without a base + filesetsWithBase = filter (fileset: ! fileset._internalIsEmptyWithoutBase) filesets; + + # The first fileset that has a base. + # This value is only accessed if there are at all. + firstWithBase = head filesetsWithBase; # To be able to union filesetTree's together, they need to have the same base path. # Base paths can be unioned by taking their common prefix, @@ -332,14 +371,14 @@ rec { # so this cannot cause a stack overflow due to a build-up of unevaluated thunks. commonBaseComponents = foldl' (components: el: commonPrefix components el._internalBaseComponents) - first._internalBaseComponents + firstWithBase._internalBaseComponents # We could also not do the `tail` here to avoid a list allocation, # but then we'd have to pay for a potentially expensive # but unnecessary `commonPrefix` call - (tail filesets); + (tail filesetsWithBase); # The common base path assembled from a filesystem root and the common components - commonBase = append first._internalBaseRoot (join commonBaseComponents); + commonBase = append firstWithBase._internalBaseRoot (join commonBaseComponents); # A list of filesetTree's that all have the same base path # This is achieved by nesting the trees into the components they have over the common base path @@ -351,14 +390,18 @@ rec { setAttrByPath (drop (length commonBaseComponents) fileset._internalBaseComponents) fileset._internalTree - ) filesets; + ) filesetsWithBase; # Folds all trees together into a single one using _unionTree # We do not use a fold here because it would cause a thunk build-up # which could cause a stack overflow for a large number of trees resultTree = _unionTrees trees; in - _create commonBase resultTree; + # If there's no values with a base, we have no files + if filesetsWithBase == [ ] then + _emptyWithoutBase + else + _create commonBase resultTree; # The union of multiple filesetTree's with the same base path. # Later elements are only evaluated if necessary. diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 0ea96859e7a..d80c3942195 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -282,24 +282,27 @@ expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: ` # File sets cannot be evaluated directly expectFailure 'union ./. ./.' 'lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.' +expectFailure '_emptyWithoutBase' 'lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.' # Past versions of the internal representation are supported expectEqual '_coerce ": value" { _type = "fileset"; _internalVersion = 0; _internalBase = ./.; }' \ - '{ _internalBase = ./.; _internalBaseComponents = path.subpath.components (path.splitRoot ./.).subpath; _internalBaseRoot = /.; _internalVersion = 2; _type = "fileset"; }' + '{ _internalBase = ./.; _internalBaseComponents = path.subpath.components (path.splitRoot ./.).subpath; _internalBaseRoot = /.; _internalIsEmptyWithoutBase = false; _internalVersion = 3; _type = "fileset"; }' expectEqual '_coerce ": value" { _type = "fileset"; _internalVersion = 1; }' \ - '{ _type = "fileset"; _internalVersion = 2; }' + '{ _type = "fileset"; _internalIsEmptyWithoutBase = false; _internalVersion = 3; }' +expectEqual '_coerce ": value" { _type = "fileset"; _internalVersion = 2; }' \ + '{ _type = "fileset"; _internalIsEmptyWithoutBase = false; _internalVersion = 3; }' # Future versions of the internal representation are unsupported -expectFailure '_coerce ": value" { _type = "fileset"; _internalVersion = 3; }' ': value is a file set created from a future version of the file set library with a different internal representation: -\s*- Internal version of the file set: 3 -\s*- Internal version of the library: 2 +expectFailure '_coerce ": value" { _type = "fileset"; _internalVersion = 4; }' ': value is a file set created from a future version of the file set library with a different internal representation: +\s*- Internal version of the file set: 4 +\s*- Internal version of the library: 3 \s*Make sure to update your Nixpkgs to have a newer version of `lib.fileset`.' # _create followed by _coerce should give the inputs back without any validation expectEqual '{ inherit (_coerce "" (_create ./. "directory")) _internalVersion _internalBase _internalTree; -}' '{ _internalBase = ./.; _internalTree = "directory"; _internalVersion = 2; }' +}' '{ _internalBase = ./.; _internalTree = "directory"; _internalVersion = 3; }' #### Resulting store path #### @@ -311,6 +314,12 @@ tree=( ) checkFileset './.' +# The empty value without a base should also result in an empty result +tree=( + [a]=0 +) +checkFileset '_emptyWithoutBase' + # Directories recursively containing no files are not included tree=( [e/]=0 @@ -408,13 +417,30 @@ expectFailure 'toSource { root = ./.; fileset = unions [ ./. ./b ]; }' 'lib.file # unions needs a list with at least 1 element expectFailure 'toSource { root = ./.; fileset = unions null; }' 'lib.fileset.unions: Expected argument to be a list, but got a null.' -expectFailure 'toSource { root = ./.; fileset = unions [ ]; }' 'lib.fileset.unions: Expected argument to be a list with at least one element, but it contains no elements.' # The tree of later arguments should not be evaluated if a former argument already includes all files tree=() checkFileset 'union ./. (_create ./. (abort "This should not be used!"))' checkFileset 'unions [ ./. (_create ./. (abort "This should not be used!")) ]' +# unions doesn't include any files for an empty list or only empty values without a base +tree=( + [x]=0 + [y/z]=0 +) +checkFileset 'unions [ ]' +checkFileset 'unions [ _emptyWithoutBase ]' +checkFileset 'unions [ _emptyWithoutBase _emptyWithoutBase ]' +checkFileset 'union _emptyWithoutBase _emptyWithoutBase' + +# The empty value without a base is the left and right identity of union +tree=( + [x]=1 + [y/z]=0 +) +checkFileset 'union ./x _emptyWithoutBase' +checkFileset 'union _emptyWithoutBase ./x' + # union doesn't include files that weren't specified tree=( [x]=1 -- cgit 1.4.1 From 43e4d8a6a1402c67420e7bad3ba8237c5f3ed327 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 3 Oct 2023 18:50:03 +0200 Subject: lib.fileset: Link to user documentation from contributor docs --- lib/fileset/README.md | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 2ef7b81e885..76c1e8d3f2c 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -1,5 +1,10 @@ # File set library +This is the internal contributor documentation. +The user documentation is [in the Nixpkgs manual](https://nixos.org/manual/nixpkgs/unstable/#sec-fileset). + +## Goals + The main goal of the file set library is to be able to select local files that should be added to the Nix store. It should have the following properties: - Easy: -- cgit 1.4.1 From b3c2281219893ee422e18f9d3917e8cc8216114d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 20 Sep 2023 21:08:13 +0200 Subject: lib.fileset: Order noEval last --- lib/fileset/internal.nix | 12 ++++++++---- lib/fileset/tests.sh | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index b3dbc5b33dc..43e6dd31b71 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -114,8 +114,10 @@ rec { # The one and only! _internalIsEmptyWithoutBase = true; - # Double __ to make it be evaluated and ordered first - __noEval = throw _noEvalMessage; + # Due to alphabetical ordering, this is evaluated last, + # which makes the nix repl output nicer than if it would be ordered first. + # It also allows evaluating it strictly up to this error, which could be useful + _noEval = throw _noEvalMessage; }; # Create a fileset, see ./README.md#fileset @@ -137,8 +139,10 @@ rec { _internalBaseComponents = components parts.subpath; _internalTree = tree; - # Double __ to make it be evaluated and ordered first - __noEval = throw _noEvalMessage; + # Due to alphabetical ordering, this is evaluated last, + # which makes the nix repl output nicer than if it would be ordered first. + # It also allows evaluating it strictly up to this error, which could be useful + _noEval = throw _noEvalMessage; }; # Coerce a value to a fileset, erroring when the value cannot be coerced. diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 525b8aaa917..1b53e33073e 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -84,14 +84,14 @@ expectStorePath() { crudeUnquoteJSON <<< "$result" } -# Check that a nix expression fails to evaluate (strictly, coercing to json, read-write-mode). +# Check that a nix expression fails to evaluate (strictly, read-write-mode). # And check the received stderr against a regex # The expression has `lib.fileset` in scope. # Usage: expectFailure NIX REGEX expectFailure() { local expr=$1 local expectedErrorRegex=$2 - if result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace 2>"$tmp/stderr" \ + if result=$(nix-instantiate --eval --strict --read-write-mode --show-trace 2>"$tmp/stderr" \ --expr "$prefixExpression $expr"); then die "$expr evaluated successfully to $result, but it was expected to fail" fi -- cgit 1.4.1 From 86802e19ceffbdff6a2a80dbec2d5f14f9f40efa Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 00:59:09 +0200 Subject: lib.fileset: _simplifyTree -> _normaliseTreeFilter --- lib/fileset/internal.nix | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 43e6dd31b71..ad10b9ac0bb 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -241,22 +241,22 @@ rec { // value; /* - Simplify a filesetTree recursively: - - Replace all directories that have no files with `null` + A normalisation of a filesetTree suitable for use in builtins.path-based filtering: + - Replace all directories that have no files with `null`. This removes directories that would be empty - - Replace all directories with all files with `"directory"` + - Replace all directories with all files with `"directory"`. This speeds up the source filter function Note that this function is strict, it evaluates the entire tree Type: Path -> filesetTree -> filesetTree */ - _simplifyTree = path: tree: + _normaliseTreeFilter = path: tree: if tree == "directory" || isAttrs tree then let entries = _directoryEntries path tree; - simpleSubtrees = mapAttrs (name: _simplifyTree (path + "/${name}")) entries; - subtreeValues = attrValues simpleSubtrees; + normalisedSubtrees = mapAttrs (name: _normaliseTreeFilter (path + "/${name}")) entries; + subtreeValues = attrValues normalisedSubtrees; in # This triggers either when all files in a directory are filtered out # Or when the directory doesn't contain any files at all @@ -266,7 +266,7 @@ rec { else if all isString subtreeValues then "directory" else - simpleSubtrees + normalisedSubtrees else tree; @@ -277,7 +277,7 @@ rec { let # Simplify the tree, necessary to make sure all empty directories are null # which has the effect that they aren't included in the result - tree = _simplifyTree fileset._internalBase fileset._internalTree; + tree = _normaliseTreeFilter fileset._internalBase fileset._internalTree; # The base path as a string with a single trailing slash baseString = -- cgit 1.4.1 From d5743d926818a3e8cb43158c827d4aff409bf12c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 3 Oct 2023 21:56:53 +0200 Subject: lib.fileset.unions: Fix outdated docs Since 4f35f003e6e5b800be75e3985054e5fce2dea50a unions supports empty lists too --- lib/fileset/default.nix | 1 - lib/fileset/tests.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index ab26112c947..f7d957bd264 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -258,7 +258,6 @@ If a directory does not recursively contain any file, it is omitted from the sto */ unions = # A list of file sets. - # Must contain at least 1 element. # The elements can also be paths, # which get [implicitly coerced to file sets](#sec-fileset-path-coercion). filesets: diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 525b8aaa917..7fdbb3ede04 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -415,7 +415,7 @@ expectFailure 'toSource { root = ./.; fileset = union ./. ./b; }' 'lib.fileset.u expectFailure 'toSource { root = ./.; fileset = unions [ ./a ./. ]; }' 'lib.fileset.unions: element 0 \('"$work"'/a\) does not exist.' expectFailure 'toSource { root = ./.; fileset = unions [ ./. ./b ]; }' 'lib.fileset.unions: element 1 \('"$work"'/b\) does not exist.' -# unions needs a list with at least 1 element +# unions needs a list expectFailure 'toSource { root = ./.; fileset = unions null; }' 'lib.fileset.unions: Expected argument to be a list, but got a null.' # The tree of later arguments should not be evaluated if a former argument already includes all files -- cgit 1.4.1 From 79dc25b48dde3af3a4b21323713cd21f90803fa7 Mon Sep 17 00:00:00 2001 From: Weijia Wang <9713184+wegank@users.noreply.github.com> Date: Thu, 21 Sep 2023 00:55:10 +0200 Subject: licenses: add inria-zelus --- lib/licenses.nix | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index 0fd64108537..980cce8712b 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -615,6 +615,12 @@ in mkLicense lset) ({ free = false; }; + inria-zelus = { + fullName = "INRIA Non-Commercial License Agreement for the Zélus compiler"; + url = "https://github.com/INRIA/zelus/raw/829f2b97cba93b0543a9ca0272269e6b8fdad356/LICENSE"; + free = false; + }; + ipa = { spdxId = "IPA"; fullName = "IPA Font License"; -- cgit 1.4.1 From efbcf5938fca42f004e34c3f56f9ab1a12a95559 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 01:00:59 +0200 Subject: lib.fileset: Add internal helpers for pretty-printing --- lib/fileset/internal.nix | 109 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index ad10b9ac0bb..ebfe69b010d 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -7,11 +7,14 @@ let isString pathExists readDir - typeOf + seq split + trace + typeOf ; inherit (lib.attrsets) + attrNames attrValues mapAttrs setAttrByPath @@ -241,7 +244,7 @@ rec { // value; /* - A normalisation of a filesetTree suitable for use in builtins.path-based filtering: + A normalisation of a filesetTree suitable filtering with `builtins.path`: - Replace all directories that have no files with `null`. This removes directories that would be empty - Replace all directories with all files with `"directory"`. @@ -270,6 +273,108 @@ rec { else tree; + /* + A minimal normalisation of a filesetTree, intended for pretty-printing: + - If all children of a path are recursively included or empty directories, the path itself is also recursively included + - If all children of a path are fully excluded or empty directories, the path itself is an empty directory + - Other empty directories are represented with the special "emptyDir" string + While these could be replaced with `null`, that would take another mapAttrs + + Note that this function is partially lazy. + + Type: Path -> filesetTree -> filesetTree (with "emptyDir"'s) + */ + _normaliseTreeMinimal = path: tree: + if tree == "directory" || isAttrs tree then + let + entries = _directoryEntries path tree; + normalisedSubtrees = mapAttrs (name: _normaliseTreeMinimal (path + "/${name}")) entries; + subtreeValues = attrValues normalisedSubtrees; + in + # If there are no entries, or all entries are empty directories, return "emptyDir". + # After this branch we know that there's at least one file + if all (value: value == "emptyDir") subtreeValues then + "emptyDir" + + # If all subtrees are fully included or empty directories + # (both of which are coincidentally represented as strings), return "directory". + # This takes advantage of the fact that empty directories can be represented as included directories. + # Note that the tree == "directory" check allows avoiding recursion + else if tree == "directory" || all (value: isString value) subtreeValues then + "directory" + + # If all subtrees are fully excluded or empty directories, return null. + # This takes advantage of the fact that empty directories can be represented as excluded directories + else if all (value: isNull value || value == "emptyDir") subtreeValues then + null + + # Mix of included and excluded entries + else + normalisedSubtrees + else + tree; + + # Trace a filesetTree in a pretty way when the resulting value is evaluated. + # This can handle both normal filesetTree's, and ones returned from _normaliseTreeMinimal + # Type: Path -> filesetTree (with "emptyDir"'s) -> Null + _printMinimalTree = base: tree: + let + treeSuffix = tree: + if isAttrs tree then + "" + else if tree == "directory" then + " (all files in directory)" + else + # This does "leak" the file type strings of the internal representation, + # but this is the main reason these file type strings even are in the representation! + # TODO: Consider removing that information from the internal representation for performance. + # The file types can still be printed by querying them only during tracing + " (${tree})"; + + # Only for attribute set trees + traceTreeAttrs = prevLine: indent: tree: + foldl' (prevLine: name: + let + subtree = tree.${name}; + + # Evaluating this prints the line for this subtree + thisLine = + trace "${indent}- ${name}${treeSuffix subtree}" prevLine; + in + if subtree == null || subtree == "emptyDir" then + # Don't print anything at all if this subtree is empty + prevLine + else if isAttrs subtree then + # A directory with explicit entries + # Do print this node, but also recurse + traceTreeAttrs thisLine "${indent} " subtree + else + # Either a file, or a recursively included directory + # Do print this node but no further recursion needed + thisLine + ) prevLine (attrNames tree); + + # Evaluating this will print the first line + firstLine = + if tree == null || tree == "emptyDir" then + trace "(empty)" null + else + trace "${toString base}${treeSuffix tree}" null; + in + if isAttrs tree then + traceTreeAttrs firstLine "" tree + else + firstLine; + + # Pretty-print a file set in a pretty way when the resulting value is evaluated + # Type: fileset -> Null + _printFileset = fileset: + if fileset._internalIsEmptyWithoutBase then + trace "(empty)" null + else + _printMinimalTree fileset._internalBase + (_normaliseTreeMinimal fileset._internalBase fileset._internalTree); + # Turn a fileset into a source filter function suitable for `builtins.path` # Only directories recursively containing at least one files are recursed into # Type: Path -> fileset -> (String -> String -> Bool) -- cgit 1.4.1 From ac2c8d321c879aa36046d404661e8a3e9c7a93fe Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 01:24:27 +0200 Subject: lib.fileset: Make expectEqual check more --- lib/fileset/tests.sh | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 1b53e33073e..13e984806c7 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -57,18 +57,35 @@ with lib.fileset;' expectEqual() { local actualExpr=$1 local expectedExpr=$2 - if ! actualResult=$(nix-instantiate --eval --strict --show-trace \ + if actualResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/actualStderr \ --expr "$prefixExpression ($actualExpr)"); then - die "$actualExpr failed to evaluate, but it was expected to succeed" + actualExitCode=$? + else + actualExitCode=$? fi - if ! expectedResult=$(nix-instantiate --eval --strict --show-trace \ + actualStderr=$(< "$tmp"/actualStderr) + + if expectedResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/expectedStderr \ --expr "$prefixExpression ($expectedExpr)"); then - die "$expectedExpr failed to evaluate, but it was expected to succeed" + expectedExitCode=$? + else + expectedExitCode=$? + fi + expectedStderr=$(< "$tmp"/expectedStderr) + + if [[ "$actualExitCode" != "$expectedExitCode" ]]; then + echo "$actualStderr" >&2 + echo "$actualResult" >&2 + die "$actualExpr should have exited with $expectedExitCode, but it exited with $actualExitCode" fi if [[ "$actualResult" != "$expectedResult" ]]; then die "$actualExpr should have evaluated to $expectedExpr:\n$expectedResult\n\nbut it evaluated to\n$actualResult" fi + + if [[ "$actualStderr" != "$expectedStderr" ]]; then + die "$actualExpr should have had this on stderr:\n$expectedStderr\n\nbut it was\n$actualStderr" + fi } # Check that a nix expression evaluates successfully to a store path and returns it (without quotes). -- cgit 1.4.1 From 467e428f0061bc25841ab7e555aee25bfa24c39b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 01:01:45 +0200 Subject: lib.fileset.trace: init --- lib/fileset/default.nix | 37 ++++++++++++ lib/fileset/tests.sh | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index ab26112c947..8eee7cf7000 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -6,12 +6,14 @@ let _coerceMany _toSourceFilter _unionMany + _printFileset ; inherit (builtins) isList isPath pathExists + seq typeOf ; @@ -275,4 +277,39 @@ If a directory does not recursively contain any file, it is omitted from the sto _unionMany ]; + /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + Type: + trace :: FileSet -> Any -> Any + + Example: + trace (unions [ ./Makefile ./src ./tests/run.sh ]) null + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + null + */ + trace = + /* + The file set to trace. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.trace: argument" fileset; + in + seq + (_printFileset actualFileset) + (x: x); + } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 13e984806c7..1b70ad433d9 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -118,6 +118,24 @@ expectFailure() { fi } +# Check that the traces of a Nix expression are as expected when evaluated. +# The expression has `lib.fileset` in scope. +# Usage: expectTrace NIX STR +expectTrace() { + local expr=$1 + local expectedTrace=$2 + + nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTrace \ + --expr "$prefixExpression trace ($expr)" || true + + actualTrace=$(sed -n 's/^trace: //p' "$tmp/stderrTrace") + + if [[ "$actualTrace" != "$expectedTrace" ]]; then + cat "$tmp"/stderrTrace >&2 + die "$expr should have traced this:\n\n$expectedTrace\n\nbut this was actually traced:\n\n$actualTrace" + fi +} + # We conditionally use inotifywait in checkFileset. # Check early whether it's available # TODO: Darwin support, though not crucial since we have Linux CI @@ -518,6 +536,139 @@ done # So, just using 1000 files for now. checkFileset 'unions (mapAttrsToList (name: _: ./. + "/${name}/a") (builtins.readDir ./.))' +## Tracing + +# The second trace argument is returned +expectEqual 'trace ./. "some value"' 'builtins.trace "(empty)" "some value"' + +# The tracing happens before the final argument is needed +expectEqual 'trace ./.' 'builtins.trace "(empty)" (x: x)' + +# Tracing an empty directory shows it as such +expectTrace './.' '(empty)' + +# This also works if there are directories, but all recursively without files +mkdir -p a/b/c +expectTrace './.' '(empty)' +rm -rf -- * + +# The empty file set without a base also prints as empty +expectTrace '_emptyWithoutBase' '(empty)' +expectTrace 'unions [ ]' '(empty)' + +# If a directory is fully included, print it as such +touch a +expectTrace './.' "$work"' (all files in directory)' +rm -rf -- * + +# If a directory is not fully included, recurse +mkdir a b +touch a/{x,y} b/{x,y} +expectTrace 'union ./a/x ./b' "$work"' +- a + - x (regular) +- b (all files in directory)' +rm -rf -- * + +# If an included path is a file, print its type +touch a x +ln -s a b +mkfifo c +expectTrace 'unions [ ./a ./b ./c ]' "$work"' +- a (regular) +- b (symlink) +- c (unknown)' +rm -rf -- * + +# Do not print directories without any files recursively +mkdir -p a/b/c +touch b x +expectTrace 'unions [ ./a ./b ]' "$work"' +- b (regular)' +rm -rf -- * + +# If all children are either fully included or empty directories, +# the parent should be printed as fully included +touch a +mkdir b +expectTrace 'union ./a ./b' "$work"' (all files in directory)' +rm -rf -- * + +mkdir -p x/b x/c +touch x/a +touch a +# If all children are either fully excluded or empty directories, +# the parent should be shown (or rather not shown) as fully excluded +expectTrace 'unions [ ./a ./x/b ./x/c ]' "$work"' +- a (regular)' +rm -rf -- * + +# Completely filtered out directories also print as empty +touch a +expectTrace '_create ./. {}' '(empty)' +rm -rf -- * + +# A general test to make sure the resulting format makes sense +# Such as indentation and ordering +mkdir -p bar/{qux,someDir} +touch bar/{baz,qux,someDir/a} foo +touch bar/qux/x +ln -s x bar/qux/a +mkfifo bar/qux/b +expectTrace 'unions [ + ./bar/baz + ./bar/qux/a + ./bar/qux/b + ./bar/someDir/a + ./foo +]' "$work"' +- bar + - baz (regular) + - qux + - a (symlink) + - b (unknown) + - someDir (all files in directory) +- foo (regular)' +rm -rf -- * + +# For recursively included directories, +# `(all files in directory)` should only be used if there's at least one file (otherwise it would be `(empty)`) +# and this should be determined without doing a full search +# +# Create a 100 level deep path, which would cause a stack overflow with the below limit +# if recursed into to figure out if the current directory is empty +mkdir -p "b/$(seq -s/ 100)" +# But that can be avoided by short-circuiting if the file a (here intentionally ordered before b) is checked first. +# In a more realistic scenario, some directories might need to be recursed into, +# but a file would be quickly found to trigger the short-circuit. +touch a +( + # Locally limit the stack to 100 * 1024 bytes, this would cause a stack overflow if the short-circuiting isn't implemented + ulimit -s 100 + expectTrace './.' "$work"' (all files in directory)' +) +rm -rf -- * + +# Partially included directories trace entries as they are evaluated +touch a b c +expectTrace '_create ./. { a = null; b = "regular"; c = throw "b"; }' "$work"' +- b (regular)' + +# Except entries that need to be evaluated to even figure out if it's only partially included: +# Here the directory could be fully excluded or included just from seeing a and b, +# so c needs to be evaluated before anything can be traced +expectTrace '_create ./. { a = null; b = null; c = throw "c"; }' '' +expectTrace '_create ./. { a = "regular"; b = "regular"; c = throw "c"; }' '' +rm -rf -- * + +# We can trace large directories (10000 here) without any problems +filesToCreate=({0..9}{0..9}{0..9}{0..9}) +expectedTrace=$work$'\n'$(printf -- '- %s (regular)\n' "${filesToCreate[@]}") +# We need an excluded file so it doesn't print as `(all files in directory)` +touch 0 "${filesToCreate[@]}" +expectTrace 'unions (mapAttrsToList (n: _: ./. + "/${n}") (removeAttrs (builtins.readDir ./.) [ "0" ]))' "$expectedTrace" +rm -rf -- * + # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets echo >&2 tests ok -- cgit 1.4.1 From 704452f29cc632f1d9fbc651993eed5d69ee8549 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 21 Sep 2023 01:43:39 +0200 Subject: lib.fileset.traceVal: init --- lib/fileset/default.nix | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/fileset/tests.sh | 14 +++++++++++++ 2 files changed, 68 insertions(+) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 8eee7cf7000..eb3b8dcb1d9 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -282,6 +282,12 @@ If a directory does not recursively contain any file, it is omitted from the sto This function is only intended for debugging purposes. The exact tracing format is unspecified and may change. + This function takes a final argument to return. + In comparison, [`traceVal`](#function-library-lib.fileset.traceVal) returns + the given file set argument. + + This variant is useful for tracing file sets in the Nix repl. + Type: trace :: FileSet -> Any -> Any @@ -312,4 +318,52 @@ If a directory does not recursively contain any file, it is omitted from the sto (_printFileset actualFileset) (x: x); + /* + Incrementally evaluate and trace a file set in a pretty way. + This function is only intended for debugging purposes. + The exact tracing format is unspecified and may change. + + This function returns the given file set. + In comparison, [`trace`](#function-library-lib.fileset.trace) takes another argument to return. + + This variant is useful for tracing file sets passed as arguments to other functions. + + Type: + traceVal :: FileSet -> FileSet + + Example: + toSource { + root = ./.; + fileset = traceVal (unions [ + ./Makefile + ./src + ./tests/run.sh + ]); + } + => + trace: /home/user/src/myProject + trace: - Makefile (regular) + trace: - src (all files in directory) + trace: - tests + trace: - run.sh (regular) + "/nix/store/...-source" + */ + traceVal = + /* + The file set to trace and return. + + This argument can also be a path, + which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + */ + fileset: + let + # "fileset" would be a better name, but that would clash with the argument name, + # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 + actualFileset = _coerce "lib.fileset.traceVal: argument" fileset; + in + seq + (_printFileset actualFileset) + # We could also return the original fileset argument here, + # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again + actualFileset; } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 1b70ad433d9..80fa21961ff 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -130,6 +130,17 @@ expectTrace() { actualTrace=$(sed -n 's/^trace: //p' "$tmp/stderrTrace") + nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTraceVal \ + --expr "$prefixExpression traceVal ($expr)" || true + + actualTraceVal=$(sed -n 's/^trace: //p' "$tmp/stderrTraceVal") + + # Test that traceVal returns the same trace as trace + if [[ "$actualTrace" != "$actualTraceVal" ]]; then + cat "$tmp"/stderrTrace >&2 + die "$expr traced this for lib.fileset.trace:\n\n$actualTrace\n\nand something different for lib.fileset.traceVal:\n\n$actualTraceVal" + fi + if [[ "$actualTrace" != "$expectedTrace" ]]; then cat "$tmp"/stderrTrace >&2 die "$expr should have traced this:\n\n$expectedTrace\n\nbut this was actually traced:\n\n$actualTrace" @@ -541,6 +552,9 @@ checkFileset 'unions (mapAttrsToList (name: _: ./. + "/${name}/a") (builtins.rea # The second trace argument is returned expectEqual 'trace ./. "some value"' 'builtins.trace "(empty)" "some value"' +# The fileset traceVal argument is returned +expectEqual 'traceVal ./.' 'builtins.trace "(empty)" (_create ./. "directory")' + # The tracing happens before the final argument is needed expectEqual 'trace ./.' 'builtins.trace "(empty)" (x: x)' -- cgit 1.4.1 From 5bbe67bf67b5728bf2c0ed8228853c661a39606d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 26 Sep 2023 19:12:58 +0200 Subject: lib.fileset: Mention trace functions in the file set evaluation error --- lib/fileset/README.md | 1 - lib/fileset/internal.nix | 4 +++- lib/fileset/tests.sh | 8 ++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 2ef7b81e885..d4c80e9433e 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -207,6 +207,5 @@ Here's a list of places in the library that need to be updated in the future: - > The file set library is currently somewhat limited but is being expanded to include more functions over time. in [the manual](../../doc/functions/fileset.section.md) -- Once a tracing function exists, `__noEval` in [internal.nix](./internal.nix) should mention it - If/Once a function to convert `lib.sources` values into file sets exists, the `_coerce` and `toSource` functions should be updated to mention that function in the error when such a value is passed - If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index ebfe69b010d..d18e37e9d6e 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -106,7 +106,9 @@ rec { ]; _noEvalMessage = '' - lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.''; + lib.fileset: Directly evaluating a file set is not supported. + To turn it into a usable source, use `lib.fileset.toSource`. + To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.''; # The empty file set without a base path _emptyWithoutBase = { diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 80fa21961ff..6c6379d0734 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -327,8 +327,12 @@ expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.to expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) does not exist.' # File sets cannot be evaluated directly -expectFailure 'union ./. ./.' 'lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.' -expectFailure '_emptyWithoutBase' 'lib.fileset: Directly evaluating a file set is not supported. Use `lib.fileset.toSource` to turn it into a usable source instead.' +expectFailure 'union ./. ./.' 'lib.fileset: Directly evaluating a file set is not supported. +\s*To turn it into a usable source, use `lib.fileset.toSource`. +\s*To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.' +expectFailure '_emptyWithoutBase' 'lib.fileset: Directly evaluating a file set is not supported. +\s*To turn it into a usable source, use `lib.fileset.toSource`. +\s*To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.' # Past versions of the internal representation are supported expectEqual '_coerce ": value" { _type = "fileset"; _internalVersion = 0; _internalBase = ./.; }' \ -- cgit 1.4.1 From 692b75257e415f46e477f473e65779b80a6e125c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 2 Oct 2023 18:49:24 +0200 Subject: lib.fileset: Refactor inotify testing to be reusable --- lib/fileset/tests.sh | 138 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 51 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 6c6379d0734..d1780af79e6 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -147,16 +147,83 @@ expectTrace() { fi } -# We conditionally use inotifywait in checkFileset. +# We conditionally use inotifywait in withFileMonitor. # Check early whether it's available # TODO: Darwin support, though not crucial since we have Linux CI if type inotifywait 2>/dev/null >/dev/null; then - canMonitorFiles=1 + canMonitor=1 else - echo "Warning: Not checking that excluded files don't get accessed since inotifywait is not available" >&2 - canMonitorFiles= + echo "Warning: Cannot check for paths not getting read since the inotifywait command (from the inotify-tools package) is not available" >&2 + canMonitor= fi +# Run a function while monitoring that it doesn't read certain paths +# Usage: withFileMonitor FUNNAME PATH... +# - FUNNAME should be a bash function that: +# - Performs some operation that should not read some paths +# - Delete the paths it shouldn't read without triggering any open events +# - PATH... are the paths that should not get read +# +# This function outputs the same as FUNNAME +withFileMonitor() { + local funName=$1 + shift + + # If we can't monitor files or have none to monitor, just run the function directly + if [[ -z "$canMonitor" ]] || (( "$#" == 0 )); then + "$funName" + else + + # Use a subshell to start the coprocess in and use a trap to kill it when exiting the subshell + ( + # Assigned by coproc, makes shellcheck happy + local watcher watcher_PID + + # Start inotifywait in the background to monitor all excluded paths + coproc watcher { + # inotifywait outputs a string on stderr when ready + # Redirect it to stdout so we can access it from the coproc's stdout fd + # exec so that the coprocess is inotify itself, making the kill below work correctly + # See below why we listen to both open and delete_self events + exec inotifywait --format='%e %w' --event open,delete_self --monitor "$@" 2>&1 + } + + # This will trigger when this subshell exits, no matter if successful or not + # After exiting the subshell, the parent shell will continue executing + trap 'kill "${watcher_PID}"' exit + + # Synchronously wait until inotifywait is ready + while read -r -u "${watcher[0]}" line && [[ "$line" != "Watches established." ]]; do + : + done + + # Call the function that should not read the given paths and delete them afterwards + "$funName" + + # Get the first event + read -r -u "${watcher[0]}" event file + + # With funName potentially reading files first before deleting them, + # there's only these two possible event timelines: + # - open*, ..., open*, delete_self, ..., delete_self: If some excluded paths were read + # - delete_self, ..., delete_self: If no excluded paths were read + # So by looking at the first event we can figure out which one it is! + # This also means we don't have to wait to collect all events. + case "$event" in + OPEN*) + die "$funName opened excluded file $file when it shouldn't have" + ;; + DELETE_SELF) + # Expected events + ;; + *) + die "During $funName, Unexpected event type '$event' on file $file that should be excluded" + ;; + esac + ) + fi +} + # Check whether a file set includes/excludes declared paths as expected, usage: # # tree=( @@ -166,7 +233,7 @@ fi # ) # checkFileset './a' # Pass the fileset as the argument declare -A tree -checkFileset() ( +checkFileset() { # New subshell so that we can have a separate trap handler, see `trap` below local fileset=$1 @@ -214,54 +281,21 @@ checkFileset() ( touch "${filesToCreate[@]}" fi - # Start inotifywait in the background to monitor all excluded files (if any) - if [[ -n "$canMonitorFiles" ]] && (( "${#excludedFiles[@]}" != 0 )); then - coproc watcher { - # inotifywait outputs a string on stderr when ready - # Redirect it to stdout so we can access it from the coproc's stdout fd - # exec so that the coprocess is inotify itself, making the kill below work correctly - # See below why we listen to both open and delete_self events - exec inotifywait --format='%e %w' --event open,delete_self --monitor "${excludedFiles[@]}" 2>&1 - } - # This will trigger when this subshell exits, no matter if successful or not - # After exiting the subshell, the parent shell will continue executing - # shellcheck disable=SC2154 - trap 'kill "${watcher_PID}"' exit - - # Synchronously wait until inotifywait is ready - while read -r -u "${watcher[0]}" line && [[ "$line" != "Watches established." ]]; do - : - done - fi - - # Call toSource with the fileset, triggering open events for all files that are added to the store expression="toSource { root = ./.; fileset = $fileset; }" - storePath=$(expectStorePath "$expression") - # Remove all files immediately after, triggering delete_self events for all of them - rm -rf -- * + # We don't have lambda's in bash unfortunately, + # so we just define a function instead and then pass its name + # shellcheck disable=SC2317 + run() { + # Call toSource with the fileset, triggering open events for all files that are added to the store + expectStorePath "$expression" + if (( ${#excludedFiles[@]} != 0 )); then + rm "${excludedFiles[@]}" + fi + } - # Only check for the inotify events if we actually started inotify earlier - if [[ -v watcher ]]; then - # Get the first event - read -r -u "${watcher[0]}" event file - - # There's only these two possible event timelines: - # - open, ..., open, delete_self, ..., delete_self: If some excluded files were read - # - delete_self, ..., delete_self: If no excluded files were read - # So by looking at the first event we can figure out which one it is! - case "$event" in - OPEN) - die "$expression opened excluded file $file when it shouldn't have" - ;; - DELETE_SELF) - # Expected events - ;; - *) - die "Unexpected event type '$event' on file $file that should be excluded" - ;; - esac - fi + # Runs the function while checking that the given excluded files aren't read + storePath=$(withFileMonitor run "${excludedFiles[@]}") # For each path that should be included, make sure it does occur in the resulting store path for p in "${included[@]}"; do @@ -276,7 +310,9 @@ checkFileset() ( die "$expression included path $p when it shouldn't have" fi done -) + + rm -rf -- * +} #### Error messages ##### -- cgit 1.4.1 From 5b4e53a30074a98621681142b4698f7c45ebc716 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 2 Oct 2023 18:49:50 +0200 Subject: lib.fileset: Don't use ulimit for testing tracing --- lib/fileset/tests.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index d1780af79e6..2dec97ab953 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -689,18 +689,23 @@ rm -rf -- * # `(all files in directory)` should only be used if there's at least one file (otherwise it would be `(empty)`) # and this should be determined without doing a full search # -# Create a 100 level deep path, which would cause a stack overflow with the below limit -# if recursed into to figure out if the current directory is empty -mkdir -p "b/$(seq -s/ 100)" -# But that can be avoided by short-circuiting if the file a (here intentionally ordered before b) is checked first. +# a is intentionally ordered first here in order to allow triggering the short-circuit behavior +# We then check that b is not read # In a more realistic scenario, some directories might need to be recursed into, # but a file would be quickly found to trigger the short-circuit. touch a -( - # Locally limit the stack to 100 * 1024 bytes, this would cause a stack overflow if the short-circuiting isn't implemented - ulimit -s 100 +mkdir b +# We don't have lambda's in bash unfortunately, +# so we just define a function instead and then pass its name +# shellcheck disable=SC2317 +run() { + # This shouldn't read b/ expectTrace './.' "$work"' (all files in directory)' -) + # Remove all files immediately after, triggering delete_self events for all of them + rmdir b +} +# Runs the function while checking that b isn't read +withFileMonitor run b rm -rf -- * # Partially included directories trace entries as they are evaluated -- cgit 1.4.1 From 2541635c1377528f7503f65537fbf96d96aa9d22 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 4 Oct 2023 23:19:56 +0200 Subject: lib.fileset: Refactor for performance and future re-use --- lib/fileset/internal.nix | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index d18e37e9d6e..6d8165691e1 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -461,6 +461,22 @@ rec { else nonEmpty; + # Transforms the filesetTree of a file set to a shorter base path, e.g. + # _shortenTreeBase [ "foo" ] (_create /foo/bar null) + # => { bar = null; } + _shortenTreeBase = targetBaseComponents: fileset: + let + recurse = index: + # If we haven't reached the required depth yet + if index < length fileset._internalBaseComponents then + # Create an attribute set and recurse as the value, this can be lazily evaluated this way + { ${elemAt fileset._internalBaseComponents index} = recurse (index + 1); } + else + # Otherwise we reached the appropriate depth, here's the original tree + fileset._internalTree; + in + recurse (length targetBaseComponents); + # Computes the union of a list of filesets. # The filesets must already be coerced and validated to be in the same filesystem root # Type: [ Fileset ] -> Fileset @@ -497,11 +513,7 @@ rec { # So the tree under `/foo/bar` gets nested under `{ bar = ...; ... }`, # while the tree under `/foo/baz` gets nested under `{ baz = ...; ... }` # Therefore allowing combined operations over them. - trees = map (fileset: - setAttrByPath - (drop (length commonBaseComponents) fileset._internalBaseComponents) - fileset._internalTree - ) filesetsWithBase; + trees = map (_shortenTreeBase commonBaseComponents) filesetsWithBase; # Folds all trees together into a single one using _unionTree # We do not use a fold here because it would cause a thunk build-up -- cgit 1.4.1 From 7843484dc16fe4bf70b64920604ece2989b8011c Mon Sep 17 00:00:00 2001 From: mirrorwitch Date: Tue, 3 Oct 2023 23:50:48 +0200 Subject: licenses: add Anti-Capitalist Software License v1.4 --- lib/licenses.nix | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index b9d6dc44e6d..d9555ca66cb 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -30,6 +30,14 @@ in mkLicense lset) ({ fullName = "Abstyles License"; }; + acsl14 = { + fullName = "Anti-Capitalist Software License v1.4"; + url = "https://anticapitalist.software/"; + /* restrictions on corporations apply for both use and redistribution */ + free = false; + redistributable = false; + }; + afl20 = { spdxId = "AFL-2.0"; fullName = "Academic Free License v2.0"; -- cgit 1.4.1 From 19241f5618f73974a3bd79853f5869c4f1dde537 Mon Sep 17 00:00:00 2001 From: Anselm Schüler Date: Fri, 6 Oct 2023 12:43:26 +0200 Subject: lib/options: correct and improve documentation of mkPackageOption --- lib/options.nix | 49 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/options.nix b/lib/options.nix index c42bc1e6c67..7491055933f 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -108,8 +108,8 @@ rec { package a module should use for some purpose. The package is specified in the third argument under `default` as a list of strings - representing its attribute path in nixpkgs (or another package set). - Because of this, you need to pass nixpkgs itself (or a subset) as the first argument. + representing its attribute path in nixpkgs. + Because of this, you need to pass nixpkgs itself as the first argument. The second argument may be either a string or a list of strings. It provides the display name of the package in the description of the generated option @@ -118,43 +118,66 @@ rec { To include extra information in the description, pass `extraDescription` to append arbitrary text to the generated description. + You can also pass an `example` value, either a literal string or an attribute path. - The default argument can be omitted if the provided name is - an attribute of pkgs (if name is a string) or a - valid attribute path in pkgs (if name is a list). + The `default` argument can be omitted if the provided name is + an attribute of pkgs (if `name` is a string) or a valid attribute path in pkgs (if `name` is a list). + You can also set `default` to just a string in which case it is interpreted as an attribute name + (a singleton attribute path, if you will). If you wish to explicitly provide no default, pass `null` as `default`. - Type: mkPackageOption :: pkgs -> (string|[string]) -> { default? :: [string], example? :: null|string|[string], extraDescription? :: string } -> option + If you want users to be able to set no package, pass `nullable = true`. + In this mode a `default = null` will not be interpreted as no default and is interpreted literally. + + Type: mkPackageOption :: pkgs -> (string|[string]) -> { nullable? :: bool, default? :: string|[string], example? :: null|string|[string], extraDescription? :: string } -> option Example: mkPackageOption pkgs "hello" { } - => { _type = "option"; default = «derivation /nix/store/3r2vg51hlxj3cx5vscp0vkv60bqxkaq0-hello-2.10.drv»; defaultText = { ... }; description = "The hello package to use."; type = { ... }; } + => { ...; default = pkgs.hello; defaultText = literalExpression "pkgs.hello"; description = "The hello package to use."; type = package; } Example: mkPackageOption pkgs "GHC" { default = [ "ghc" ]; example = "pkgs.haskell.packages.ghc92.ghc.withPackages (hkgs: [ hkgs.primes ])"; } - => { _type = "option"; default = «derivation /nix/store/jxx55cxsjrf8kyh3fp2ya17q99w7541r-ghc-8.10.7.drv»; defaultText = { ... }; description = "The GHC package to use."; example = { ... }; type = { ... }; } + => { ...; default = pkgs.ghc; defaultText = literalExpression "pkgs.ghc"; description = "The GHC package to use."; example = literalExpression "pkgs.haskell.packages.ghc92.ghc.withPackages (hkgs: [ hkgs.primes ])"; type = package; } Example: - mkPackageOption pkgs [ "python39Packages" "pytorch" ] { + mkPackageOption pkgs [ "python3Packages" "pytorch" ] { extraDescription = "This is an example and doesn't actually do anything."; } - => { _type = "option"; default = «derivation /nix/store/gvqgsnc4fif9whvwd9ppa568yxbkmvk8-python3.9-pytorch-1.10.2.drv»; defaultText = { ... }; description = "The pytorch package to use. This is an example and doesn't actually do anything."; type = { ... }; } + => { ...; default = pkgs.python3Packages.pytorch; defaultText = literalExpression "pkgs.python3Packages.pytorch"; description = "The pytorch package to use. This is an example and doesn't actually do anything."; type = package; } + + Example: + mkPackageOption pkgs "nushell" { + nullable = true; + } + => { ...; default = pkgs.nushell; defaultText = literalExpression "pkgs.nushell"; description = "The nushell package to use."; type = nullOr package; } + Example: + mkPackageOption pkgs "coreutils" { + default = null; + } + => { ...; description = "The coreutils package to use."; type = package; } + + Example: + mkPackageOption pkgs "dbus" { + nullable = true; + default = null; + } + => { ...; default = null; description = "The dbus package to use."; type = nullOr package; } */ mkPackageOption = - # Package set (a specific version of nixpkgs or a subset) + # Package set (an instantiation of nixpkgs such as pkgs in modules) pkgs: # Name for the package, shown in option description name: { - # Whether the package can be null, for example to disable installing a package altogether. + # Whether the package can be null, for example to disable installing a package altogether (defaults to false) nullable ? false, - # The attribute path where the default package is located (may be omitted) + # The attribute path where the default package is located (may be omitted, in which case it is copied from `name`) default ? name, # A string or an attribute path to use as an example (may be omitted) example ? null, -- cgit 1.4.1 From d27d38c13a8ec4e2a6adf314148986f2f7741a40 Mon Sep 17 00:00:00 2001 From: Anselm Schüler Date: Fri, 6 Oct 2023 13:28:46 +0200 Subject: lib/tests: add more tests for mkPackageOption --- lib/tests/modules.sh | 6 ++++++ lib/tests/modules/declare-mkPackageOption.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) (limited to 'lib') diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 05c99e6de83..c2f1a426a34 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -227,8 +227,14 @@ checkConfigOutput '^false$' config.enableAlias ./alias-with-priority-can-overrid # Check mkPackageOption checkConfigOutput '^"hello"$' config.package.pname ./declare-mkPackageOption.nix +checkConfigOutput '^"hello"$' config.namedPackage.pname ./declare-mkPackageOption.nix +checkConfigOutput '^".*Hello.*"$' options.namedPackage.description ./declare-mkPackageOption.nix +checkConfigOutput '^"hello"$' config.pathPackage.pname ./declare-mkPackageOption.nix +checkConfigOutput '^"pkgs\.hello\.override \{ stdenv = pkgs\.clangStdenv; \}"$' options.packageWithExample.example.text ./declare-mkPackageOption.nix +checkConfigOutput '^".*Example extra description\..*"$' options.packageWithExtraDescription.description ./declare-mkPackageOption.nix checkConfigError 'The option .undefinedPackage. is used but not defined' config.undefinedPackage ./declare-mkPackageOption.nix checkConfigOutput '^null$' config.nullablePackage ./declare-mkPackageOption.nix +checkConfigOutput '^"null or package"$' options.nullablePackageWithDefault.type.description ./declare-mkPackageOption.nix # submoduleWith diff --git a/lib/tests/modules/declare-mkPackageOption.nix b/lib/tests/modules/declare-mkPackageOption.nix index 640b19a7bf2..37f070467b4 100644 --- a/lib/tests/modules/declare-mkPackageOption.nix +++ b/lib/tests/modules/declare-mkPackageOption.nix @@ -7,6 +7,28 @@ in { options = { package = lib.mkPackageOption pkgs "hello" { }; + namedPackage = lib.mkPackageOption pkgs "Hello" { + default = [ "hello" ]; + }; + + namedPackageSingletonDefault = lib.mkPackageOption pkgs "Hello" { + default = "hello"; + }; + + pathPackage = lib.mkPackageOption pkgs [ "hello" ] { }; + + packageWithExample = lib.mkPackageOption pkgs "hello" { + example = "pkgs.hello.override { stdenv = pkgs.clangStdenv; }"; + }; + + packageWithPathExample = lib.mkPackageOption pkgs "hello" { + example = [ "hello" ]; + }; + + packageWithExtraDescription = lib.mkPackageOption pkgs "hello" { + extraDescription = "Example extra description."; + }; + undefinedPackage = lib.mkPackageOption pkgs "hello" { default = null; }; @@ -15,5 +37,9 @@ in { nullable = true; default = null; }; + + nullablePackageWithDefault = lib.mkPackageOption pkgs "hello" { + nullable = true; + }; }; } -- cgit 1.4.1 From f07537da75e5ece7d223a1cfa7e59ad1825be2a7 Mon Sep 17 00:00:00 2001 From: Anselm Schüler Date: Fri, 6 Oct 2023 12:44:48 +0200 Subject: lib/options: add pkgsText parameter to mkPackageOption --- lib/options.nix | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/options.nix b/lib/options.nix index 7491055933f..30fc61c22db 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -108,8 +108,14 @@ rec { package a module should use for some purpose. The package is specified in the third argument under `default` as a list of strings - representing its attribute path in nixpkgs. - Because of this, you need to pass nixpkgs itself as the first argument. + representing its attribute path in nixpkgs (or another package set). + Because of this, you need to pass nixpkgs itself (usually `pkgs` in a module; + alternatively to nixpkgs itself, another package set) as the first argument. + + If you pass another package set you should set the `pkgsText` option. + This option is used to display the expression for the package set. It is `"pkgs"` by default. + If your expression is complex you should parenthesize it, as the `pkgsText` argument + is usually immediately followed by an attribute lookup (`.`). The second argument may be either a string or a list of strings. It provides the display name of the package in the description of the generated option @@ -131,7 +137,7 @@ rec { If you want users to be able to set no package, pass `nullable = true`. In this mode a `default = null` will not be interpreted as no default and is interpreted literally. - Type: mkPackageOption :: pkgs -> (string|[string]) -> { nullable? :: bool, default? :: string|[string], example? :: null|string|[string], extraDescription? :: string } -> option + Type: mkPackageOption :: pkgs -> (string|[string]) -> { nullable? :: bool, default? :: string|[string], example? :: null|string|[string], extraDescription? :: string, pkgsText? :: string } -> option Example: mkPackageOption pkgs "hello" { } @@ -168,9 +174,16 @@ rec { default = null; } => { ...; default = null; description = "The dbus package to use."; type = nullOr package; } + + Example: + mkPackageOption pkgs.javaPackages "OpenJFX" { + default = "openjfx20"; + pkgsText = "pkgs.javaPackages"; + } + => { ...; default = pkgs.javaPackages.openjfx20; defaultText = literalExpression "pkgs.javaPackages.openjfx20"; description = "The OpenJFX package to use."; type = package; } */ mkPackageOption = - # Package set (an instantiation of nixpkgs such as pkgs in modules) + # Package set (an instantiation of nixpkgs such as pkgs in modules or another package set) pkgs: # Name for the package, shown in option description name: @@ -183,6 +196,8 @@ rec { example ? null, # Additional text to include in the option description (may be omitted) extraDescription ? "", + # Representation of the package set passed as pkgs (defaults to `"pkgs"`) + pkgsText ? "pkgs" }: let name' = if isList name then last name else name; @@ -194,15 +209,15 @@ rec { default' = if isList default then default else [ default ]; defaultPath = concatStringsSep "." default'; defaultValue = attrByPath default' - (throw "${defaultPath} cannot be found in pkgs") pkgs; + (throw "${defaultPath} cannot be found in ${pkgsText}") pkgs; in { default = defaultValue; - defaultText = literalExpression ("pkgs." + defaultPath); + defaultText = literalExpression ("${pkgsText}." + defaultPath); } else if nullable then { default = null; } else { }) // lib.optionalAttrs (example != null) { example = literalExpression - (if isList example then "pkgs." + concatStringsSep "." example else example); + (if isList example then "${pkgsText}." + concatStringsSep "." example else example); }); /* Alias of mkPackageOption. Previously used to create options with markdown -- cgit 1.4.1 From 0de25e25b06591af5bb2362256d4cecbe29eb556 Mon Sep 17 00:00:00 2001 From: Anderson Torres Date: Tue, 3 Oct 2023 00:06:05 -0300 Subject: lib: refactor - rename `(enable|with)Feature(As|)` inputs to more descriptive names --- lib/strings.nix | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/strings.nix b/lib/strings.nix index d7642ce10fa..628669d86bb 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -854,7 +854,7 @@ rec { assert (lib.isBool flag); mesonOption feature (if flag then "enabled" else "disabled"); - /* Create an --{enable,disable}- string that can be passed to + /* Create an --{enable,disable}- string that can be passed to standard GNU Autoconf scripts. Example: @@ -863,11 +863,12 @@ rec { enableFeature false "shared" => "--disable-shared" */ - enableFeature = enable: feat: - assert isString feat; # e.g. passing openssl instead of "openssl" - "--${if enable then "enable" else "disable"}-${feat}"; + enableFeature = flag: feature: + assert lib.isBool flag; + assert lib.isString feature; # e.g. passing openssl instead of "openssl" + "--${if flag then "enable" else "disable"}-${feature}"; - /* Create an --{enable-=,disable-} string that can be passed to + /* Create an --{enable-=,disable-} string that can be passed to standard GNU Autoconf scripts. Example: @@ -876,9 +877,10 @@ rec { enableFeatureAs false "shared" (throw "ignored") => "--disable-shared" */ - enableFeatureAs = enable: feat: value: enableFeature enable feat + optionalString enable "=${value}"; + enableFeatureAs = flag: feature: value: + enableFeature flag feature + optionalString flag "=${value}"; - /* Create an --{with,without}- string that can be passed to + /* Create an --{with,without}- string that can be passed to standard GNU Autoconf scripts. Example: @@ -887,11 +889,11 @@ rec { withFeature false "shared" => "--without-shared" */ - withFeature = with_: feat: - assert isString feat; # e.g. passing openssl instead of "openssl" - "--${if with_ then "with" else "without"}-${feat}"; + withFeature = flag: feature: + assert isString feature; # e.g. passing openssl instead of "openssl" + "--${if flag then "with" else "without"}-${feature}"; - /* Create an --{with-=,without-} string that can be passed to + /* Create an --{with-=,without-} string that can be passed to standard GNU Autoconf scripts. Example: @@ -900,7 +902,8 @@ rec { withFeatureAs false "shared" (throw "ignored") => "--without-shared" */ - withFeatureAs = with_: feat: value: withFeature with_ feat + optionalString with_ "=${value}"; + withFeatureAs = flag: feature: value: + withFeature flag feature + optionalString flag "=${value}"; /* Create a fixed width string with additional prefix to match required width. -- cgit 1.4.1 From 61fa1255f86bd12d81f81dc93391ee7365059647 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 23:57:03 +0200 Subject: link to documentation on IFD in the Nix manual --- doc/languages-frameworks/rust.section.md | 2 +- lib/fileset/README.md | 2 +- pkgs/README.md | 12 ++---------- 3 files changed, 4 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index 67e23cc74d4..3bd8e1c7651 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -817,7 +817,7 @@ $ cargo test ## Using community maintained Rust toolchains {#using-community-maintained-rust-toolchains} ::: {.note} -Note: The following projects cannot be used within nixpkgs since [IFD](#ssec-import-from-derivation) is disallowed. +The following projects cannot be used within Nixpkgs since [Import From Derivation](https://nixos.org/manual/nix/unstable/language/import-from-derivation) (IFD) is disallowed in Nixpkgs. To package things that require Rust nightly, `RUSTC_BOOTSTRAP = true;` can sometimes be used as a hack. ::: diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 1aed7efec4c..d2215803692 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -176,7 +176,7 @@ File sets do not support Nix store paths in strings such as `"/nix/store/...-sou Arguments: - (+) Such paths are usually produced by derivations, which means `toSource` would either: - - Require IFD if `builtins.path` is used as the underlying primitive + - Require [Import From Derivation](https://nixos.org/manual/nix/unstable/language/import-from-derivation) (IFD) if `builtins.path` is used as the underlying primitive - Require importing the entire `root` into the store such that derivations can be used to do the filtering - (+) The convenient path coercion like `union ./foo ./bar` wouldn't work for absolute paths, requiring more verbose alternate interfaces: - `let root = "/nix/store/...-source"; in union "${root}/foo" "${root}/bar"` diff --git a/pkgs/README.md b/pkgs/README.md index 5cc70247581..bfa7e611255 100644 --- a/pkgs/README.md +++ b/pkgs/README.md @@ -368,19 +368,11 @@ See the Nixpkgs manual for more details on [standard meta-attributes](https://ni ### Import From Derivation -Import From Derivation (IFD) is disallowed in Nixpkgs for performance reasons: -[Hydra] evaluates the entire package set, and sequential builds during evaluation would increase evaluation times to become impractical. - -[Hydra]: https://github.com/NixOS/hydra +[Import From Derivation](https://nixos.org/manual/nix/unstable/language/import-from-derivation) (IFD) is disallowed in Nixpkgs for performance reasons: +[Hydra](https://github.com/NixOS/hydra) evaluates the entire package set, and sequential builds during evaluation would increase evaluation times to become impractical. Import From Derivation can be worked around in some cases by committing generated intermediate files to version control and reading those instead. - - -See also [NixOS Wiki: Import From Derivation]. - -[NixOS Wiki: Import From Derivation]: https://nixos.wiki/wiki/Import_From_Derivation - ## Sources ### Fetching Sources -- cgit 1.4.1 From 55ab538abf0a476ad731ed7411063d966f832439 Mon Sep 17 00:00:00 2001 From: Anselm Schüler Date: Fri, 6 Oct 2023 13:32:49 +0200 Subject: lib/tests: add test for pkgsText parameter and package set selection of mkPackageOption --- lib/tests/modules.sh | 2 ++ lib/tests/modules/declare-mkPackageOption.nix | 8 ++++++++ 2 files changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index c2f1a426a34..21d4978a116 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -235,6 +235,8 @@ checkConfigOutput '^".*Example extra description\..*"$' options.packageWithExtra checkConfigError 'The option .undefinedPackage. is used but not defined' config.undefinedPackage ./declare-mkPackageOption.nix checkConfigOutput '^null$' config.nullablePackage ./declare-mkPackageOption.nix checkConfigOutput '^"null or package"$' options.nullablePackageWithDefault.type.description ./declare-mkPackageOption.nix +checkConfigOutput '^"myPkgs\.hello"$' options.packageWithPkgsText.defaultText.text ./declare-mkPackageOption.nix +checkConfigOutput '^"hello-other"$' options.packageFromOtherSet.default.pname ./declare-mkPackageOption.nix # submoduleWith diff --git a/lib/tests/modules/declare-mkPackageOption.nix b/lib/tests/modules/declare-mkPackageOption.nix index 37f070467b4..e13e68447e0 100644 --- a/lib/tests/modules/declare-mkPackageOption.nix +++ b/lib/tests/modules/declare-mkPackageOption.nix @@ -41,5 +41,13 @@ in { nullablePackageWithDefault = lib.mkPackageOption pkgs "hello" { nullable = true; }; + + packageWithPkgsText = lib.mkPackageOption pkgs "hello" { + pkgsText = "myPkgs"; + }; + + packageFromOtherSet = let myPkgs = { + hello = pkgs.hello // { pname = "hello-other"; }; + }; in lib.mkPackageOption myPkgs "hello" { }; }; } -- cgit 1.4.1 From 7e24b3619f6f7180988931d6743d2c7f2f5b8713 Mon Sep 17 00:00:00 2001 From: Anselm Schüler Date: Fri, 6 Oct 2023 12:51:23 +0200 Subject: lib/options: refactor mkPackageOption --- lib/options.nix | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/options.nix b/lib/options.nix index 30fc61c22db..7821924873d 100644 --- a/lib/options.nix +++ b/lib/options.nix @@ -201,21 +201,21 @@ rec { }: let name' = if isList name then last name else name; - in mkOption ({ - type = with lib.types; (if nullable then nullOr else lib.id) package; - description = "The ${name'} package to use." - + (if extraDescription == "" then "" else " ") + extraDescription; - } // (if default != null then let default' = if isList default then default else [ default ]; - defaultPath = concatStringsSep "." default'; + defaultText = concatStringsSep "." default'; defaultValue = attrByPath default' - (throw "${defaultPath} cannot be found in ${pkgsText}") pkgs; - in { - default = defaultValue; - defaultText = literalExpression ("${pkgsText}." + defaultPath); - } else if nullable then { - default = null; - } else { }) // lib.optionalAttrs (example != null) { + (throw "${defaultText} cannot be found in ${pkgsText}") pkgs; + defaults = if default != null then { + default = defaultValue; + defaultText = literalExpression ("${pkgsText}." + defaultText); + } else optionalAttrs nullable { + default = null; + }; + in mkOption (defaults // { + description = "The ${name'} package to use." + + (if extraDescription == "" then "" else " ") + extraDescription; + type = with lib.types; (if nullable then nullOr else lib.id) package; + } // optionalAttrs (example != null) { example = literalExpression (if isList example then "${pkgsText}." + concatStringsSep "." example else example); }); -- cgit 1.4.1 From d70633f91cb27d9314940d3a6e9385f89bf7f007 Mon Sep 17 00:00:00 2001 From: Oliver Schmidt Date: Sun, 10 Sep 2023 17:56:29 +0200 Subject: lib.attrsets.attrsToList: add function For transforming back between lists and attrsets, it makes sense to have a quasi-inverse of `builtins.listToAttrs` available as a library function. Co-authored-by: Silvan Mosberger Co-authored-by: Robert Hensing --- lib/attrsets.nix | 30 ++++++++++++++++++++++++++++++ lib/default.nix | 4 ++-- lib/tests/misc.nix | 24 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/attrsets.nix b/lib/attrsets.nix index 77e36d3271f..09a2efa40a2 100644 --- a/lib/attrsets.nix +++ b/lib/attrsets.nix @@ -542,6 +542,36 @@ rec { attrs: map (name: f name attrs.${name}) (attrNames attrs); + /* + Deconstruct an attrset to a list of name-value pairs as expected by [`builtins.listToAttrs`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-listToAttrs). + Each element of the resulting list is an attribute set with these attributes: + - `name` (string): The name of the attribute + - `value` (any): The value of the attribute + + The following is always true: + ```nix + builtins.listToAttrs (attrsToList attrs) == attrs + ``` + + :::{.warning} + The opposite is not always true. In general expect that + ```nix + attrsToList (builtins.listToAttrs list) != list + ``` + + This is because the `listToAttrs` removes duplicate names and doesn't preserve the order of the list. + ::: + + Example: + attrsToList { foo = 1; bar = "asdf"; } + => [ { name = "bar"; value = "asdf"; } { name = "foo"; value = 1; } ] + + Type: + attrsToList :: AttrSet -> [ { name :: String; value :: Any; } ] + + */ + attrsToList = mapAttrsToList nameValuePair; + /* Like `mapAttrs`, except that it recursively applies itself to the *leaf* attributes of a potentially-nested attribute set: diff --git a/lib/default.nix b/lib/default.nix index e4bf45aac3b..282aa8f61e0 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -81,8 +81,8 @@ let inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath getAttrFromPath attrVals attrValues getAttrs catAttrs filterAttrs filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs - mapAttrs' mapAttrsToList concatMapAttrs mapAttrsRecursive mapAttrsRecursiveCond - genAttrs isDerivation toDerivation optionalAttrs + mapAttrs' mapAttrsToList attrsToList concatMapAttrs mapAttrsRecursive + mapAttrsRecursiveCond genAttrs isDerivation toDerivation optionalAttrs zipAttrsWithNames zipAttrsWith zipAttrs recursiveUpdateUntil recursiveUpdate matchAttrs overrideExisting showAttrPath getOutput getBin getLib getDev getMan chooseDevOutputs zipWithNames zip diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 6d55ae68477..702be9529b4 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -20,6 +20,10 @@ let expr = (builtins.tryEval (builtins.seq expr "didn't throw")); expected = { success = false; value = false; }; }; + testingEval = expr: { + expr = (builtins.tryEval expr).success; + expected = true; + }; testingDeepThrow = expr: testingThrow (builtins.deepSeq expr expr); testSanitizeDerivationName = { name, expected }: @@ -784,6 +788,26 @@ runTests { expected = { a = 1; b = 2; }; }; + testListAttrsReverse = let + exampleAttrs = {foo=1; bar="asdf"; baz = [1 3 3 7]; fnord=null;}; + exampleSingletonList = [{name="foo"; value=1;}]; + in { + expr = { + isReverseToListToAttrs = builtins.listToAttrs (attrsToList exampleAttrs) == exampleAttrs; + isReverseToAttrsToList = attrsToList (builtins.listToAttrs exampleSingletonList) == exampleSingletonList; + testDuplicatePruningBehaviour = attrsToList (builtins.listToAttrs [{name="a"; value=2;} {name="a"; value=1;}]); + }; + expected = { + isReverseToAttrsToList = true; + isReverseToListToAttrs = true; + testDuplicatePruningBehaviour = [{name="a"; value=2;}]; + }; + }; + + testAttrsToListsCanDealWithFunctions = testingEval ( + attrsToList { someFunc= a: a + 1;} + ); + # GENERATORS # these tests assume attributes are converted to lists # in alphabetical order -- cgit 1.4.1 From 4ecf0258149f4e197e9dea4d71ab22756ffc5ece Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 26 Sep 2023 02:10:46 +0200 Subject: lib.fileset.intersection: init --- lib/fileset/README.md | 29 ++++++++++++++- lib/fileset/default.nix | 41 +++++++++++++++++++++ lib/fileset/internal.nix | 93 +++++++++++++++++++++++++++++++++++++++++++++++ lib/fileset/tests.sh | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 1aed7efec4c..393cafb42e3 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -143,6 +143,33 @@ Arguments: - (-) Leaves us with no identity element for `union` and no reasonable return value for `unions []`. From a set theory perspective, which has a well-known notion of empty sets, this is unintuitive. +### No intersection for lists + +While there is `intersection a b`, there is no function `intersections [ a b c ]`. + +Arguments: +- (+) There is no known use case for such a function, it can be added later if a use case arises +- (+) There is no suitable return value for `intersections [ ]`, see also "Nullary intersections" [here](https://en.wikipedia.org/w/index.php?title=List_of_set_identities_and_relations&oldid=1177174035#Definitions) + - (-) Could throw an error for that case + - (-) Create a special value to represent "all the files" and return that + - (+) Such a value could then not be used with `fileFilter` unless the internal representation is changed considerably + - (-) Could return the empty file set + - (+) This would be wrong in set theory +- (-) Inconsistent with `union` and `unions` + +### Intersection base path + +The base path of the result of an `intersection` is the longest base path of the arguments. +E.g. the base path of `intersection ./foo ./foo/bar` is `./foo/bar`. +Meanwhile `intersection ./foo ./bar` returns the empty file set without a base path. + +Arguments: +- Alternative: Use the common prefix of all base paths as the resulting base path + - (-) This is unnecessarily strict, because the purpose of the base path is to track the directory under which files _could_ be in the file set. It should be as long as possible. + All files contained in `intersection ./foo ./foo/bar` will be under `./foo/bar` (never just under `./foo`), and `intersection ./foo ./bar` will never contain any files (never under `./.`). + This would lead to `toSource` having to unexpectedly throw errors for cases such as `toSource { root = ./foo; fileset = intersect ./foo base; }`, where `base` may be `./bar` or `./.`. + - (-) There is no benefit to the user, since base path is not directly exposed in the interface + ### Empty directories File sets can only represent a _set_ of local files, directories on their own are not representable. @@ -161,7 +188,7 @@ Arguments: - `./.` represents all files in `./.` _and_ the directory itself, but not its subdirectories, meaning that at least `./.` will be preserved even if it's empty. - In that case, `intersect ./. ./foo` should only include files and no directories themselves, since `./.` includes only `./.` as a directory, and same for `./foo`, so there's no overlap in directories. + In that case, `intersection ./. ./foo` should only include files and no directories themselves, since `./.` includes only `./.` as a directory, and same for `./foo`, so there's no overlap in directories. But intuitively this operation should result in the same as `./foo` – everything else is just confusing. - (+) This matches how Git only supports files, so developers should already be used to it. - (-) Empty directories (even if they contain nested directories) are neither representable nor preserved when coercing from paths. diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 93a552262b0..7bd70167038 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -7,6 +7,7 @@ let _toSourceFilter _unionMany _printFileset + _intersection ; inherit (builtins) @@ -18,6 +19,7 @@ let ; inherit (lib.lists) + elemAt imap0 ; @@ -276,6 +278,45 @@ If a directory does not recursively contain any file, it is omitted from the sto _unionMany ]; + /* + The file set containing all files that are in both of two given file sets. + See also [Intersection (set theory)](https://en.wikipedia.org/wiki/Intersection_(set_theory)). + + The given file sets are evaluated as lazily as possible, + with the first argument being evaluated first if needed. + + Type: + intersection :: FileSet -> FileSet -> FileSet + + Example: + # Limit the selected files to the ones in ./., so only ./src and ./Makefile + intersection ./. (unions [ ../LICENSE ./src ./Makefile ]) + */ + intersection = + # The first file set. + # This argument can also be a path, + # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + fileset1: + # The second file set. + # This argument can also be a path, + # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + fileset2: + let + filesets = _coerceMany "lib.fileset.intersection" [ + { + context = "first argument"; + value = fileset1; + } + { + context = "second argument"; + value = fileset2; + } + ]; + in + _intersection + (elemAt filesets 0) + (elemAt filesets 1); + /* Incrementally evaluate and trace a file set in a pretty way. This function is only intended for debugging purposes. diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 6d8165691e1..546b93f158a 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -477,6 +477,27 @@ rec { in recurse (length targetBaseComponents); + # Transforms the filesetTree of a file set to a longer base path, e.g. + # _lengthenTreeBase [ "foo" "bar" ] (_create /foo { bar.baz = "regular"; }) + # => { baz = "regular"; } + _lengthenTreeBase = targetBaseComponents: fileset: + let + recurse = index: tree: + # If the filesetTree is an attribute set and we haven't reached the required depth yet + if isAttrs tree && index < length targetBaseComponents then + # Recurse with the tree under the right component (which might not exist) + recurse (index + 1) (tree.${elemAt targetBaseComponents index} or null) + else + # For all values here we can just return the tree itself: + # tree == null -> the result is also null, everything is excluded + # tree == "directory" -> the result is also "directory", + # because the base path is always a directory and everything is included + # isAttrs tree -> the result is `tree` + # because we don't need to recurse any more since `index == length longestBaseComponents` + tree; + in + recurse (length fileset._internalBaseComponents) fileset._internalTree; + # Computes the union of a list of filesets. # The filesets must already be coerced and validated to be in the same filesystem root # Type: [ Fileset ] -> Fileset @@ -545,4 +566,76 @@ rec { # The non-null elements have to be attribute sets representing partial trees # We need to recurse into those zipAttrsWith (name: _unionTrees) withoutNull; + + # Computes the intersection of a list of filesets. + # The filesets must already be coerced and validated to be in the same filesystem root + # Type: Fileset -> Fileset -> Fileset + _intersection = fileset1: fileset2: + let + # The common base components prefix, e.g. + # (/foo/bar, /foo/bar/baz) -> /foo/bar + # (/foo/bar, /foo/baz) -> /foo + commonBaseComponentsLength = + # TODO: Have a `lib.lists.commonPrefixLength` function such that we don't need the list allocation from commonPrefix here + length ( + commonPrefix + fileset1._internalBaseComponents + fileset2._internalBaseComponents + ); + + # To be able to intersect filesetTree's together, they need to have the same base path. + # Base paths can be intersected by taking the longest one (if any) + + # The fileset with the longest base, if any, e.g. + # (/foo/bar, /foo/bar/baz) -> /foo/bar/baz + # (/foo/bar, /foo/baz) -> null + longestBaseFileset = + if commonBaseComponentsLength == length fileset1._internalBaseComponents then + # The common prefix is the same as the first path, so the second path is equal or longer + fileset2 + else if commonBaseComponentsLength == length fileset2._internalBaseComponents then + # The common prefix is the same as the second path, so the first path is longer + fileset1 + else + # The common prefix is neither the first nor the second path + # This means there's no overlap between the two sets + null; + + # Whether the result should be the empty value without a base + resultIsEmptyWithoutBase = + # If either fileset is the empty fileset without a base, the intersection is too + fileset1._internalIsEmptyWithoutBase + || fileset2._internalIsEmptyWithoutBase + # If there is no overlap between the base paths + || longestBaseFileset == null; + + # Lengthen each fileset's tree to the longest base prefix + tree1 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset1; + tree2 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset2; + + # With two filesetTree's with the same base, we can compute their intersection + resultTree = _intersectTree tree1 tree2; + in + if resultIsEmptyWithoutBase then + _emptyWithoutBase + else + _create longestBaseFileset._internalBase resultTree; + + # The intersection of two filesetTree's with the same base path + # The second element is only evaluated as much as necessary. + # Type: filesetTree -> filesetTree -> filesetTree + _intersectTree = lhs: rhs: + if isAttrs lhs && isAttrs rhs then + # Both sides are attribute sets, we can recurse for the attributes existing on both sides + mapAttrs + (name: _intersectTree lhs.${name}) + (builtins.intersectAttrs lhs rhs) + else if lhs == null || isString rhs then + # If the lhs is null, the result should also be null + # And if the rhs is the identity element + # (a string, aka it includes everything), then it's also the lhs + lhs + else + # In all other cases it's the rhs + rhs; } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 9e09da80924..7a104654983 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -587,6 +587,97 @@ done # So, just using 1000 files for now. checkFileset 'unions (mapAttrsToList (name: _: ./. + "/${name}/a") (builtins.readDir ./.))' + +## lib.fileset.intersection + + +# Different filesystem roots in root and fileset are not supported +mkdir -p {foo,bar}/mock-root +expectFailure 'with ((import ).extend (import )).fileset; + toSource { root = ./.; fileset = intersection ./foo/mock-root ./bar/mock-root; } +' 'lib.fileset.intersection: Filesystem roots are not the same: +\s*first argument: root "'"$work"'/foo/mock-root" +\s*second argument: root "'"$work"'/bar/mock-root" +\s*Different roots are not supported.' +rm -rf -- * + +# Coercion errors show the correct context +expectFailure 'toSource { root = ./.; fileset = intersection ./a ./.; }' 'lib.fileset.intersection: first argument \('"$work"'/a\) does not exist.' +expectFailure 'toSource { root = ./.; fileset = intersection ./. ./b; }' 'lib.fileset.intersection: second argument \('"$work"'/b\) does not exist.' + +# The tree of later arguments should not be evaluated if a former argument already excludes all files +tree=( + [a]=0 +) +checkFileset 'intersection _emptyWithoutBase (_create ./. (abort "This should not be used!"))' +# We don't have any combinators that can explicitly remove files yet, so we need to rely on internal functions to test this for now +checkFileset 'intersection (_create ./. { a = null; }) (_create ./. { a = abort "This should not be used!"; })' + +# If either side is empty, the result is empty +tree=( + [a]=0 +) +checkFileset 'intersection _emptyWithoutBase _emptyWithoutBase' +checkFileset 'intersection _emptyWithoutBase (_create ./. null)' +checkFileset 'intersection (_create ./. null) _emptyWithoutBase' +checkFileset 'intersection (_create ./. null) (_create ./. null)' + +# If the intersection base paths are not overlapping, the result is empty and has no base path +mkdir a b c +touch {a,b,c}/x +expectEqual 'toSource { root = ./c; fileset = intersection ./a ./b; }' 'toSource { root = ./c; fileset = _emptyWithoutBase; }' +rm -rf -- * + +# If the intersection exists, the resulting base path is the longest of them +mkdir a +touch x a/b +expectEqual 'toSource { root = ./a; fileset = intersection ./a ./.; }' 'toSource { root = ./a; fileset = ./a; }' +expectEqual 'toSource { root = ./a; fileset = intersection ./. ./a; }' 'toSource { root = ./a; fileset = ./a; }' +rm -rf -- * + +# Also finds the intersection with null'd filesetTree's +tree=( + [a]=0 + [b]=1 + [c]=0 +) +checkFileset 'intersection (_create ./. { a = "regular"; b = "regular"; c = null; }) (_create ./. { a = null; b = "regular"; c = "regular"; })' + +# Actually computes the intersection between files +tree=( + [a]=0 + [b]=0 + [c]=1 + [d]=1 + [e]=0 + [f]=0 +) +checkFileset 'intersection (unions [ ./a ./b ./c ./d ]) (unions [ ./c ./d ./e ./f ])' + +tree=( + [a/x]=0 + [a/y]=0 + [b/x]=1 + [b/y]=1 + [c/x]=0 + [c/y]=0 +) +checkFileset 'intersection ./b ./.' +checkFileset 'intersection ./b (unions [ ./a/x ./a/y ./b/x ./b/y ./c/x ./c/y ])' + +# Complicated case +tree=( + [a/x]=0 + [a/b/i]=1 + [c/d/x]=0 + [c/d/f]=1 + [c/x]=0 + [c/e/i]=1 + [c/e/j]=1 +) +checkFileset 'intersection (unions [ ./a/b ./c/d ./c/e ]) (unions [ ./a ./c/d/f ./c/e ])' + + ## Tracing # The second trace argument is returned @@ -609,6 +700,10 @@ rm -rf -- * # The empty file set without a base also prints as empty expectTrace '_emptyWithoutBase' '(empty)' expectTrace 'unions [ ]' '(empty)' +mkdir foo bar +touch {foo,bar}/x +expectTrace 'intersection ./foo ./bar' '(empty)' +rm -rf -- * # If a directory is fully included, print it as such touch a -- cgit 1.4.1 From 389be8db81592ce4c5519a96a459593e07eb82f7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 11 Oct 2023 16:16:40 +0200 Subject: lib.fileset: Minor contributor doc adjustments Co-authored-by: Robert Hensing --- lib/fileset/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 393cafb42e3..c439e449544 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -58,7 +58,8 @@ An attribute set with these values: - `_internalBase` (path): Any files outside of this path cannot influence the set of files. - This is always a directory. + This is always a directory and should be as long as possible. + This is used by `lib.fileset.toSource` to check that all files are under the `root` argument - `_internalBaseRoot` (path): The filesystem root of `_internalBase`, same as `(lib.path.splitRoot _internalBase).root`. @@ -172,7 +173,8 @@ Arguments: ### Empty directories -File sets can only represent a _set_ of local files, directories on their own are not representable. +File sets can only represent a _set_ of local files. +Directories on their own are not representable. Arguments: - (+) There does not seem to be a sensible set of combinators when directories can be represented on their own. -- cgit 1.4.1 From 0a10279342995a40fc4ad08a7f2a8250cc5ef4d7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 11 Oct 2023 22:56:53 +0200 Subject: lib.makeOverridable: fix functionArgs on returned function --- lib/customisation.nix | 7 ++++--- lib/tests/misc.nix | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/customisation.nix b/lib/customisation.nix index ec2513021f9..5ef4f29e6f6 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -69,8 +69,8 @@ rec { ".overrideDerivation" to learn about `overrideDerivation` and caveats related to its use. */ - makeOverridable = f: origArgs: - let + makeOverridable = f: lib.setFunctionArgs + (origArgs: let result = f origArgs; # Creates a functor with the same arguments as f @@ -95,7 +95,8 @@ rec { lib.setFunctionArgs result (lib.functionArgs result) // { override = overrideArgs; } - else result; + else result) + (lib.functionArgs f); /* Call the package function in the file `fn` with the required diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 6527c31e49c..2e7fda2b1f8 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -43,6 +43,18 @@ in runTests { +# CUSTOMIZATION + + testFunctionArgsMakeOverridable = { + expr = functionArgs (makeOverridable ({ a, b, c ? null}: {})); + expected = { a = false; b = false; c = true; }; + }; + + testFunctionArgsMakeOverridableOverride = { + expr = functionArgs (makeOverridable ({ a, b, c ? null }: {}) { a = 1; b = 2; }).override; + expected = { a = false; b = false; c = true; }; + }; + # TRIVIAL testId = { -- cgit 1.4.1 From b82c1c7b5c47f99efb2469b4c6295901a63499dc Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 12 Oct 2023 17:37:41 +0200 Subject: lib.fix: Improve doc more Done together in and after the docs team meeting Co-Authored-By: Robert Hensing --- lib/fixed-points.nix | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix index d1cb2dc030d..3444e95e15a 100644 --- a/lib/fixed-points.nix +++ b/lib/fixed-points.nix @@ -3,12 +3,16 @@ rec { /* `fix f` computes the fixed point of the given function `f`. In other words, the return value is `x` in `x = f x`. - `f` is usually returns an attribute set that expects its final, non-recursive representation as an argument. `f` must be a lazy function. + This means that `x` must be a value that can be partially evaluated, + such as an attribute set, a list, or a function. + This way, `f` can use one part of `x` to compute another part. - **How it works** + **Relation to syntactic recursion** - For context, Nix lets you define attribute set values in terms of other attributes using the `rec { }` attribute set literal syntax. + This section explains `fix` by refactoring from syntactic recursion to a call of `fix` instead. + + For context, Nix lets you define attributes in terms of other attributes syntactically using the [`rec { }` syntax](https://nixos.org/manual/nix/stable/language/constructs.html#recursive-sets). ```nix nix-repl> rec { @@ -19,7 +23,8 @@ rec { { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - This is convenient when constructing a value to pass to a function for example, but a similar effect can be achieved with a `let` binding: + This is convenient when constructing a value to pass to a function for example, + but an equivalent effect can be achieved with the `let` binding syntax: ```nix nix-repl> let self = { @@ -30,7 +35,7 @@ rec { { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - `let` bindings are nice, but as it is with `let` bindings in general, we may get more reuse out of the code by defining a function. + But in general you can get more reuse out of `let` bindings by refactoring them to a function. ```nix nix-repl> f = self: { @@ -40,27 +45,32 @@ rec { } ``` - This is where `fix` comes in. Note that the body of the `fix` function - looks a lot like our earlier `let` binding, and that's no coincidence. - Fix is no more than such a recursive `let` binding, but with everything - except the recursion factored out into a function parameter `f`. + This is where `fix` comes in, it contains the syntactic that's not in `f` anymore. ```nix - fix = f: + nix-repl> fix = f: let self = f self; in self; ``` - So applying `fix` is another way to express our earlier examples. + By applying `fix` we get the final result. - ``` + ```nix nix-repl> fix f { bar = "bar"; foo = "foo"; foobar = "foobar"; } ``` - This example did not _need_ `fix`, and arguably it shouldn't be used in such an example. - However, `fix` is useful when your `f` is a parameter, or when it is constructed from higher order functions. + Such a refactored `f` using `fix` is not useful by itself. + See [`extends`](#function-library-lib.fixedPoints.extends) for an example use case. + There `self` is also often called `final`. Type: fix :: (a -> a) -> a + + Example: + fix (self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }) + => { bar = "bar"; foo = "foo"; foobar = "foobar"; } + + fix (self: [ 1 2 (elemAt self 0 + elemAt self 1) ]) + => [ 1 2 3 ] */ fix = f: let x = f x; in x; -- cgit 1.4.1 From 7247ec07ba5d78f7ce93b89461a1503a9a1b021f Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 16 Oct 2023 15:49:53 +0200 Subject: lib.fileset: Improved error for unsupported coercion values --- lib/fileset/internal.nix | 4 ++-- lib/fileset/tests.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 546b93f158a..9892172955c 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -172,11 +172,11 @@ rec { else if ! isPath value then if isStringLike value then throw '' - ${context} ("${toString value}") is a string-like value, but it should be a path instead. + ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead. Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' else throw '' - ${context} is of type ${typeOf value}, but it should be a path instead.'' + ${context} is of type ${typeOf value}, but it should be a file set or a path instead.'' else if ! pathExists value then throw '' ${context} (${toString value}) does not exist.'' diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 7a104654983..529f23ae887 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -355,8 +355,8 @@ expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: ` rm -rf * # Path coercion only works for paths -expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a path instead.' -expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a path instead. +expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.' +expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a file set or a path instead. \s*Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' # Path coercion errors for non-existent paths -- cgit 1.4.1 From d7c49ca7152768114a9359b729557f77b8343545 Mon Sep 17 00:00:00 2001 From: Gerg-L Date: Fri, 13 Oct 2023 20:45:19 -0400 Subject: lib.getExe': check arguments --- lib/meta.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/meta.nix b/lib/meta.nix index 44730a71551..2e817c42327 100644 --- a/lib/meta.nix +++ b/lib/meta.nix @@ -162,5 +162,12 @@ rec { getExe' pkgs.imagemagick "convert" => "/nix/store/5rs48jamq7k6sal98ymj9l4k2bnwq515-imagemagick-7.1.1-15/bin/convert" */ - getExe' = x: y: "${lib.getBin x}/bin/${y}"; + getExe' = x: y: + assert lib.assertMsg (lib.isDerivation x) + "lib.meta.getExe': The first argument is of type ${builtins.typeOf x}, but it should be a derivation instead."; + assert lib.assertMsg (lib.isString y) + "lib.meta.getExe': The second argument is of type ${builtins.typeOf y}, but it should be a string instead."; + assert lib.assertMsg (builtins.length (lib.splitString "/" y) == 1) + "lib.meta.getExe': The second argument \"${y}\" is a nested path with a \"/\" character, but it should just be the name of the executable instead."; + "${lib.getBin x}/bin/${y}"; } -- cgit 1.4.1 From d2161d06321770f37fd86bfa15296dd8de0196a0 Mon Sep 17 00:00:00 2001 From: Gerg-L Date: Mon, 16 Oct 2023 18:55:38 -0400 Subject: lib/tests: add tests for getExe' and getExe --- lib/tests/misc.nix | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 2e7fda2b1f8..47853f47278 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -1906,4 +1906,32 @@ runTests { expr = (with types; either int (listOf (either bool str))).description; expected = "signed integer or list of (boolean or string)"; }; + +# Meta + testGetExe'Output = { + expr = getExe' { + type = "derivation"; + out = "somelonghash"; + bin = "somelonghash"; + } "executable"; + expected = "somelonghash/bin/executable"; + }; + + testGetExeOutput = { + expr = getExe { + type = "derivation"; + out = "somelonghash"; + bin = "somelonghash"; + meta.mainProgram = "mainProgram"; + }; + expected = "somelonghash/bin/mainProgram"; + }; + + testGetExe'FailureFirstArg = testingThrow ( + getExe' "not a derivation" "executable" + ); + + testGetExe'FailureSecondArg = testingThrow ( + getExe' { type = "derivation"; } "dir/executable" + ); } -- cgit 1.4.1 From 93b8f14a2d36a17ebbd40f3f3907e43a01d44502 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 20 Oct 2023 20:57:10 +0800 Subject: doc: lib.customization: add Type and Example tags Add the "Type:" blocks. Move the examples below the descriptions whenever possibles Add "Example:" tags before the examples moved below the descriptions. --- lib/customisation.nix | 89 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/customisation.nix b/lib/customisation.nix index 5ef4f29e6f6..ed76699a23e 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -13,16 +13,7 @@ rec { scenarios (e.g. in ~/.config/nixpkgs/config.nix). For instance, if you want to "patch" the derivation returned by a package function in Nixpkgs to build another version than what the - function itself provides, you can do something like this: - - mySed = overrideDerivation pkgs.gnused (oldAttrs: { - name = "sed-4.2.2-pre"; - src = fetchurl { - url = ftp://alpha.gnu.org/gnu/sed/sed-4.2.2-pre.tar.bz2; - hash = "sha256-MxBJRcM2rYzQYwJ5XKxhXTQByvSg5jZc5cSHEZoB2IY="; - }; - patches = []; - }); + function itself provides. For another application, see build-support/vm, where this function is used to build arbitrary derivations inside a QEMU @@ -35,6 +26,19 @@ rec { You should in general prefer `drv.overrideAttrs` over this function; see the nixpkgs manual for more information on overriding. + + Example: + mySed = overrideDerivation pkgs.gnused (oldAttrs: { + name = "sed-4.2.2-pre"; + src = fetchurl { + url = ftp://alpha.gnu.org/gnu/sed/sed-4.2.2-pre.tar.bz2; + hash = "sha256-MxBJRcM2rYzQYwJ5XKxhXTQByvSg5jZc5cSHEZoB2IY="; + }; + patches = []; + }); + + Type: + overrideDerivation :: Derivation -> ( Derivation -> AttrSet ) -> Derivation */ overrideDerivation = drv: f: let @@ -55,6 +59,11 @@ rec { injects `override` attribute which can be used to override arguments of the function. + Please refer to "Nixpkgs Contributors Guide" section + ".overrideDerivation" to learn about `overrideDerivation` and caveats + related to its use. + + Example: nix-repl> x = {a, b}: { result = a + b; } nix-repl> y = lib.makeOverridable x { a = 1; b = 2; } @@ -65,9 +74,8 @@ rec { nix-repl> y.override { a = 10; } { override = «lambda»; overrideDerivation = «lambda»; result = 12; } - Please refer to "Nixpkgs Contributors Guide" section - ".overrideDerivation" to learn about `overrideDerivation` and caveats - related to its use. + Type: + makeOverridable :: (AttrSet -> a) -> AttrSet -> a */ makeOverridable = f: lib.setFunctionArgs (origArgs: let @@ -105,20 +113,29 @@ rec { `autoArgs`. This function is intended to be partially parameterised, e.g., + ```nix callPackage = callPackageWith pkgs; pkgs = { libfoo = callPackage ./foo.nix { }; libbar = callPackage ./bar.nix { }; }; + ``` If the `libbar` function expects an argument named `libfoo`, it is automatically passed as an argument. Overrides or missing arguments can be supplied in `args`, e.g. + ```nix libbar = callPackage ./bar.nix { libfoo = null; enableX11 = true; }; + ``` + + + + Type: + callPackageWith :: AttrSet -> ((AttrSet -> a) | Path) -> AttrSet -> a */ callPackageWith = autoArgs: fn: args: let @@ -129,7 +146,7 @@ rec { # This includes automatic ones and ones passed explicitly allArgs = builtins.intersectAttrs fargs autoArgs // args; - # A list of argument names that the function requires, but + # a list of argument names that the function requires, but # wouldn't be passed to it missingArgs = lib.attrNames # Filter out arguments that have a default value @@ -176,7 +193,11 @@ rec { /* Like callPackage, but for a function that returns an attribute set of derivations. The override function is added to the - individual attributes. */ + individual attributes. + + Type: + callPackagesWith :: AttrSet -> ((AttrSet -> AttrSet) | Path) -> AttrSet -> AttrSet + */ callPackagesWith = autoArgs: fn: args: let f = if lib.isFunction fn then fn else import fn; @@ -193,7 +214,11 @@ rec { /* Add attributes to each output of a derivation without changing - the derivation itself and check a given condition when evaluating. */ + the derivation itself and check a given condition when evaluating. + + Type: + extendDerivation :: Bool -> Any -> Derivation -> Derivation + */ extendDerivation = condition: passthru: drv: let outputs = drv.outputs or [ "out" ]; @@ -227,7 +252,11 @@ rec { /* Strip a derivation of all non-essential attributes, returning only those needed by hydra-eval-jobs. Also strictly evaluate the result to ensure that there are no thunks kept alive to prevent - garbage collection. */ + garbage collection. + + Type: + hydraJob :: (Derivation | Null) -> (Derivation | Null) + */ hydraJob = drv: let outputs = drv.outputs or ["out"]; @@ -265,7 +294,11 @@ rec { called with the overridden packages. The package sets may be hierarchical: the packages in the set are called with the scope provided by `newScope` and the set provides a `newScope` attribute - which can form the parent scope for later package sets. */ + which can form the parent scope for later package sets. + + Type: + makeScope :: (AttrSet -> ((AttrSet -> a) | Path) -> AttrSet -> a) -> (AttrSet -> AttrSet) -> AttrSet + */ makeScope = newScope: f: let self = f self // { newScope = scope: newScope (self // scope); @@ -287,7 +320,25 @@ rec { { inherit otherSplices keep extra f; }; /* Like makeScope, but aims to support cross compilation. It's still ugly, but - hopefully it helps a little bit. */ + hopefully it helps a little bit. + + Type: + makeScopeWithSplicing' :: + { splicePackages :: Splice -> AttrSet + , newScope :: AttrSet -> ((AttrSet -> a) | Path) -> AttrSet -> a + } + -> { otherSplices :: Splice, keep :: AttrSet -> AttrSet, extra :: AttrSet -> AttrSet } + -> AttrSet + + Splice :: + { pkgsBuildBuild :: AttrSet + , pkgsBuildHost :: AttrSet + , pkgsBuildTarget :: AttrSet + , pkgsHostHost :: AttrSet + , pkgsHostTarget :: AttrSet + , pkgsTargetTarget :: AttrSet + } + */ makeScopeWithSplicing' = { splicePackages , newScope -- cgit 1.4.1 From 0695d3e8fe08dac78fc59354ae2f926915b0d6b6 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 21 Oct 2023 03:11:21 +0800 Subject: lib.overrideDerivation: inter-link the documentation Co-authored-by: Valentin Gagarin --- lib/customisation.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/customisation.nix b/lib/customisation.nix index ed76699a23e..61bb531d2f6 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -59,8 +59,7 @@ rec { injects `override` attribute which can be used to override arguments of the function. - Please refer to "Nixpkgs Contributors Guide" section - ".overrideDerivation" to learn about `overrideDerivation` and caveats + Please refer to documentation on [`.overrideDerivation`](#sec-pkg-overrideDerivation) to learn about `overrideDerivation` and caveats related to its use. Example: -- cgit 1.4.1 From 20b62456f0c85b206e59db5f16a7cd0016fa8262 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Wed, 7 Jun 2023 09:39:42 -0700 Subject: lib/systems/parse.nix: add, use removeAbiSuffix gnu-config will ignore the portion of a triple matching the regex `e?abi.*$` when determining the validity of a triple. In other words, `i386-linuxabichickenlips` is a valid triple. This commit updates our parsing routines to match gnu-config. I was recently surprised to discover that it is in fact possible to shoehorn ABI flavors into nix doubles in a way which preserves their property of being a (non-canonical) subset of the valid gnu-config triples. This commit is required in order to exploit that discovery to add automatic detection of ILP32 platforms (64-bit void*, 32-bit int, like the Apple Watch and MIPS n32) to Nix. --- lib/systems/parse.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/systems/parse.nix b/lib/systems/parse.nix index 34bfd94b3ce..b69ad669e18 100644 --- a/lib/systems/parse.nix +++ b/lib/systems/parse.nix @@ -29,6 +29,15 @@ let assert type.check value; setType type.name ({ inherit name; } // value)); + # gnu-config will ignore the portion of a triple matching the + # regex `e?abi.*$` when determining the validity of a triple. In + # other words, `i386-linuxabichickenlips` is a valid triple. + removeAbiSuffix = x: + let match = builtins.match "(.*)e?abi.*" x; + in if match==null + then x + else lib.elemAt match 0; + in rec { @@ -466,7 +475,7 @@ rec { else vendors.unknown; kernel = if hasPrefix "darwin" args.kernel then getKernel "darwin" else if hasPrefix "netbsd" args.kernel then getKernel "netbsd" - else getKernel args.kernel; + else getKernel (removeAbiSuffix args.kernel); abi = /**/ if args ? abi then getAbi args.abi else if isLinux parsed || isWindows parsed then -- cgit 1.4.1 From f4e0043049523dba68eba917edd2f7e322ab0c2e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 10 Oct 2023 19:10:02 +0200 Subject: lib.fileset.fileFilter: init --- lib/fileset/default.nix | 51 ++++++++++++++++++++++++++++++++++++ lib/fileset/internal.nix | 26 +++++++++++++++++++ lib/fileset/tests.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 7bd70167038..0342be3e037 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -6,6 +6,7 @@ let _coerceMany _toSourceFilter _unionMany + _fileFilter _printFileset _intersection ; @@ -41,6 +42,7 @@ let ; inherit (lib.trivial) + isFunction pipe ; @@ -278,6 +280,55 @@ If a directory does not recursively contain any file, it is omitted from the sto _unionMany ]; + /* + Filter a file set to only contain files matching some predicate. + + Type: + fileFilter :: + ({ + name :: String, + type :: String, + ... + } -> Bool) + -> FileSet + -> FileSet + + Example: + # Include all regular `default.nix` files in the current directory + fileFilter (file: file.name == "default.nix") ./. + + # Include all non-Nix files from the current directory + fileFilter (file: ! hasSuffix ".nix" file.name) ./. + + # Include all files that start with a "." in the current directory + fileFilter (file: hasPrefix "." file.name) ./. + + # Include all regular files (not symlinks or others) in the current directory + fileFilter (file: file.type == "regular") + */ + fileFilter = + /* + The predicate function to call on all files contained in given file set. + A file is included in the resulting file set if this function returns true for it. + + This function is called with an attribute set containing these attributes: + + - `name` (String): The name of the file + + - `type` (String, one of `"regular"`, `"symlink"` or `"unknown"`): The type of the file. + This matches result of calling [`builtins.readFileType`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-readFileType) on the file's path. + + Other attributes may be added in the future. + */ + predicate: + # The file set to filter based on the predicate function + fileset: + if ! isFunction predicate then + throw "lib.fileset.fileFilter: Expected the first argument to be a function, but it's a ${typeOf predicate} instead." + else + _fileFilter predicate + (_coerce "lib.fileset.fileFilter: second argument" fileset); + /* The file set containing all files that are in both of two given file sets. See also [Intersection (set theory)](https://en.wikipedia.org/wiki/Intersection_(set_theory)). diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 9892172955c..2d52a8cb410 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -638,4 +638,30 @@ rec { else # In all other cases it's the rhs rhs; + + _fileFilter = predicate: fileset: + let + recurse = path: tree: + mapAttrs (name: subtree: + if isAttrs subtree || subtree == "directory" then + recurse (path + "/${name}") subtree + else if + predicate { + inherit name; + type = subtree; + # To ensure forwards compatibility with more arguments being added in the future, + # adding an attribute which can't be deconstructed :) + "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file }:`, use `{ name, file, ... }:` instead." = null; + } + then + subtree + else + null + ) (_directoryEntries path tree); + in + if fileset._internalIsEmptyWithoutBase then + _emptyWithoutBase + else + _create fileset._internalBase + (recurse fileset._internalBase fileset._internalTree); } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 529f23ae887..d8d8dd41318 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -678,6 +678,73 @@ tree=( checkFileset 'intersection (unions [ ./a/b ./c/d ./c/e ]) (unions [ ./a ./c/d/f ./c/e ])' +## File filter + +# The predicate is not called when there's no files +tree=() +checkFileset 'fileFilter (file: abort "this is not needed") ./.' +checkFileset 'fileFilter (file: abort "this is not needed") _emptyWithoutBase' + +# The predicate must be able to handle extra attributes +touch a +expectFailure 'toSource { root = ./.; fileset = fileFilter ({ name, type }: true) ./.; }' 'called with unexpected argument '\''"lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you'\''re using `\{ name, file \}:`, use `\{ name, file, ... \}:` instead."'\' +rm -rf -- * + +# .name is the name, and it works correctly, even recursively +tree=( + [a]=1 + [b]=0 + [c/a]=1 + [c/b]=0 + [d/c/a]=1 + [d/c/b]=0 +) +checkFileset 'fileFilter (file: file.name == "a") ./.' +tree=( + [a]=0 + [b]=1 + [c/a]=0 + [c/b]=1 + [d/c/a]=0 + [d/c/b]=1 +) +checkFileset 'fileFilter (file: file.name != "a") ./.' + +# `.type` is the file type +mkdir d +touch d/a +ln -s d/b d/b +mkfifo d/c +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type == "regular") ./.; }' \ + 'toSource { root = ./.; fileset = ./d/a; }' +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type == "symlink") ./.; }' \ + 'toSource { root = ./.; fileset = ./d/b; }' +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type == "unknown") ./.; }' \ + 'toSource { root = ./.; fileset = ./d/c; }' +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type != "regular") ./.; }' \ + 'toSource { root = ./.; fileset = union ./d/b ./d/c; }' +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type != "symlink") ./.; }' \ + 'toSource { root = ./.; fileset = union ./d/a ./d/c; }' +expectEqual \ + 'toSource { root = ./.; fileset = fileFilter (file: file.type != "unknown") ./.; }' \ + 'toSource { root = ./.; fileset = union ./d/a ./d/b; }' +rm -rf -- * + +# It's lazy +tree=( + [b]=1 + [c/a]=1 +) +# Note that union evaluates the first argument first if necessary, that's why we can use ./c/a here +checkFileset 'union ./c/a (fileFilter (file: assert file.name != "a"; true) ./.)' +# but here we need to use ./c +checkFileset 'union (fileFilter (file: assert file.name != "a"; true) ./.) ./c' + ## Tracing # The second trace argument is returned -- cgit 1.4.1 From 7227cb1d734a2a3ce19a7ce94d5541e03ed1582c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 01:51:15 +0200 Subject: lib.fileset: Fix shellcheck warnings in tests --- lib/fileset/tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index d8d8dd41318..6a57177d14d 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -332,7 +332,7 @@ expectFailure 'with ((import ).extend (import ).extend (import Date: Thu, 26 Oct 2023 01:53:39 +0200 Subject: lib.fileset.toSource: Test with unknown file type Currently just throws the Nix error because unknown file types are not supported by the Nix store, but nothing catches this error earlier (yet, see next commit) --- lib/fileset/tests.sh | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 6a57177d14d..86ef1989f60 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -354,6 +354,11 @@ expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: ` \s*- Set `fileset` to a file set that cannot contain files outside the `root` \('"$work"'/a\). This could change the files included in the result.' rm -rf -- * +# non-regular and non-symlink files cannot be added to the Nix store +mkfifo a +expectFailure 'toSource { root = ./.; fileset = ./a; }' 'file '\'"$work"'/a'\'' has an unsupported type' +rm -rf -- * + # Path coercion only works for paths expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.' expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a file set or a path instead. -- cgit 1.4.1 From 0f6cc8018c7b7f9244f06ad8072ab017808ad0c2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 01:56:42 +0200 Subject: lib.fileset.toSource: Improve error for unknown file types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This does decrease performance unfortunately Benchmarking expression toSource { root = ./.; fileset = ./.; } Mean CPU time 0.103747 (σ = 0.012415) for 10 runs is 97.32181384964636% (σ = 16.34179537413021%) of the old value 0.106602 (σ = 0.0125571) Statistic .envs.elements (205920) is 105.5842% (+10891) of the old value 195029 Statistic .gc.totalBytes (20247696) is 101.7495% (+348160) of the old value 19899536 Statistic .nrThunks (134824) is 108.7878% (+10891) of the old value 123933 Statistic .symbols.number (996) is 100.1005% (+1) of the old value 995 Statistic .values.number (275238) is 104.1199% (+10891) of the old value 264347 --- lib/fileset/internal.nix | 49 +++++++++++++++++++++++++++++------------------- lib/fileset/tests.sh | 4 +++- 2 files changed, 33 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 2d52a8cb410..afaa8363373 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -424,7 +424,7 @@ rec { # Filter suited when there's some files # This can't be used for when there's no files, because the base directory is always included nonEmpty = - path: _: + path: type: let # Add a slash to the path string, turning "/foo" to "/foo/", # making sure to not have any false prefix matches below. @@ -432,26 +432,37 @@ rec { # but builtins.path doesn't call the filter function on the `path` argument itself, # meaning this function can never receive "/" as an argument pathSlash = path + "/"; + + include = + # Same as `hasPrefix pathSlash baseString`, but more efficient. + # With base /foo/bar we need to include /foo: + # hasPrefix "/foo/" "/foo/bar/" + if substring 0 (stringLength pathSlash) baseString == pathSlash then + true + # Same as `! hasPrefix baseString pathSlash`, but more efficient. + # With base /foo/bar we need to exclude /baz + # ! hasPrefix "/baz/" "/foo/bar/" + else if substring 0 baseLength pathSlash != baseString then + false + else + # Same as `removePrefix baseString path`, but more efficient. + # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe. + # We don't use pathSlash here because we only needed the trailing slash for the prefix matching. + # With base /foo and path /foo/bar/baz this gives + # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz")) + # == inTree (split "/" "bar/baz") + # == inTree [ "bar" "baz" ] + inTree (split "/" (substring baseLength (-1) path)); in - # Same as `hasPrefix pathSlash baseString`, but more efficient. - # With base /foo/bar we need to include /foo: - # hasPrefix "/foo/" "/foo/bar/" - if substring 0 (stringLength pathSlash) baseString == pathSlash then - true - # Same as `! hasPrefix baseString pathSlash`, but more efficient. - # With base /foo/bar we need to exclude /baz - # ! hasPrefix "/baz/" "/foo/bar/" - else if substring 0 baseLength pathSlash != baseString then - false + # This relies on the fact that Nix only distinguishes path types "directory", "regular", "symlink" and "unknown", + # so everything except "unknown" is allowed, seems reasonable to rely on that + if include && type == "unknown" then + throw '' + lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: ${path} + This file is neither a regular file nor a symlink, the only file types supported by the Nix store. + Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.'' else - # Same as `removePrefix baseString path`, but more efficient. - # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe. - # We don't use pathSlash here because we only needed the trailing slash for the prefix matching. - # With base /foo and path /foo/bar/baz this gives - # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz")) - # == inTree (split "/" "bar/baz") - # == inTree [ "bar" "baz" ] - inTree (split "/" (substring baseLength (-1) path)); + include; in # Special case because the code below assumes that the _internalBase is always included in the result # which shouldn't be done when we have no files at all in the base diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 86ef1989f60..5b756b8fc59 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -356,7 +356,9 @@ rm -rf -- * # non-regular and non-symlink files cannot be added to the Nix store mkfifo a -expectFailure 'toSource { root = ./.; fileset = ./a; }' 'file '\'"$work"'/a'\'' has an unsupported type' +expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: '"$work"'/a +\s*This file is neither a regular file nor a symlink, the only file types supported by the Nix store. +\s*Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.' rm -rf -- * # Path coercion only works for paths -- cgit 1.4.1 From 47c81d3286bd027d5b55fa581f9502eff4ea8822 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 02:15:54 +0200 Subject: lib.fileset.toSource: Optimise unknown file type error Compared to the parent commit, this removes any detectable performance cost :) --- lib/fileset/internal.nix | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index afaa8363373..76b95c6ae47 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -432,37 +432,38 @@ rec { # but builtins.path doesn't call the filter function on the `path` argument itself, # meaning this function can never receive "/" as an argument pathSlash = path + "/"; - - include = - # Same as `hasPrefix pathSlash baseString`, but more efficient. - # With base /foo/bar we need to include /foo: - # hasPrefix "/foo/" "/foo/bar/" - if substring 0 (stringLength pathSlash) baseString == pathSlash then - true - # Same as `! hasPrefix baseString pathSlash`, but more efficient. - # With base /foo/bar we need to exclude /baz - # ! hasPrefix "/baz/" "/foo/bar/" - else if substring 0 baseLength pathSlash != baseString then - false - else - # Same as `removePrefix baseString path`, but more efficient. - # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe. - # We don't use pathSlash here because we only needed the trailing slash for the prefix matching. - # With base /foo and path /foo/bar/baz this gives - # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz")) - # == inTree (split "/" "bar/baz") - # == inTree [ "bar" "baz" ] - inTree (split "/" (substring baseLength (-1) path)); in - # This relies on the fact that Nix only distinguishes path types "directory", "regular", "symlink" and "unknown", - # so everything except "unknown" is allowed, seems reasonable to rely on that - if include && type == "unknown" then - throw '' + ( + # Same as `hasPrefix pathSlash baseString`, but more efficient. + # With base /foo/bar we need to include /foo: + # hasPrefix "/foo/" "/foo/bar/" + if substring 0 (stringLength pathSlash) baseString == pathSlash then + true + # Same as `! hasPrefix baseString pathSlash`, but more efficient. + # With base /foo/bar we need to exclude /baz + # ! hasPrefix "/baz/" "/foo/bar/" + else if substring 0 baseLength pathSlash != baseString then + false + else + # Same as `removePrefix baseString path`, but more efficient. + # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe. + # We don't use pathSlash here because we only needed the trailing slash for the prefix matching. + # With base /foo and path /foo/bar/baz this gives + # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz")) + # == inTree (split "/" "bar/baz") + # == inTree [ "bar" "baz" ] + inTree (split "/" (substring baseLength (-1) path)) + ) + # This is a way have an additional check in case the above is true without any significant performance cost + && ( + # This relies on the fact that Nix only distinguishes path types "directory", "regular", "symlink" and "unknown", + # so everything except "unknown" is allowed, seems reasonable to rely on that + type != "unknown" + || throw '' lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: ${path} This file is neither a regular file nor a symlink, the only file types supported by the Nix store. Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.'' - else - include; + ); in # Special case because the code below assumes that the _internalBase is always included in the result # which shouldn't be done when we have no files at all in the base -- cgit 1.4.1 From 50df7f977548c296e475ad37c1d44afcdc9f8e26 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 4 Oct 2023 23:23:30 +0200 Subject: lib.fileset.difference: init --- lib/fileset/default.nix | 53 ++++++++++++++++++++++++++ lib/fileset/internal.nix | 80 +++++++++++++++++++++++++++++++++++++++ lib/fileset/tests.sh | 98 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 0342be3e037..4a97633b4a8 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -9,6 +9,7 @@ let _fileFilter _printFileset _intersection + _difference ; inherit (builtins) @@ -368,6 +369,58 @@ If a directory does not recursively contain any file, it is omitted from the sto (elemAt filesets 0) (elemAt filesets 1); + /* + The file set containing all files from the first file set that are not in the second file set. + See also [Difference (set theory)](https://en.wikipedia.org/wiki/Complement_(set_theory)#Relative_complement). + + The given file sets are evaluated as lazily as possible, + with the first argument being evaluated first if needed. + + Type: + union :: FileSet -> FileSet -> FileSet + + Example: + # Create a file set containing all files from the current directory, + # except ones under ./tests + difference ./. ./tests + + let + # A set of Nix-related files + nixFiles = unions [ ./default.nix ./nix ./tests/default.nix ]; + in + # Create a file set containing all files under ./tests, except ones in `nixFiles`, + # meaning only without ./tests/default.nix + difference ./tests nixFiles + */ + difference = + # The positive file set. + # The result can only contain files that are also in this file set. + # + # This argument can also be a path, + # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + positive: + # The negative file set. + # The result will never contain files that are also in this file set. + # + # This argument can also be a path, + # which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). + negative: + let + filesets = _coerceMany "lib.fileset.difference" [ + { + context = "first argument (positive set)"; + value = positive; + } + { + context = "second argument (negative set)"; + value = negative; + } + ]; + in + _difference + (elemAt filesets 0) + (elemAt filesets 1); + /* Incrementally evaluate and trace a file set in a pretty way. This function is only intended for debugging purposes. diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 2d52a8cb410..8543bf616c2 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -639,6 +639,86 @@ rec { # In all other cases it's the rhs rhs; + # Compute the set difference between two file sets. + # The filesets must already be coerced and validated to be in the same filesystem root. + # Type: Fileset -> Fileset -> Fileset + _difference = positive: negative: + let + # The common base components prefix, e.g. + # (/foo/bar, /foo/bar/baz) -> /foo/bar + # (/foo/bar, /foo/baz) -> /foo + commonBaseComponentsLength = + # TODO: Have a `lib.lists.commonPrefixLength` function such that we don't need the list allocation from commonPrefix here + length ( + commonPrefix + positive._internalBaseComponents + negative._internalBaseComponents + ); + + # We need filesetTree's with the same base to be able to compute the difference between them + # This here is the filesetTree from the negative file set, but for a base path that matches the positive file set. + # Examples: + # For `difference /foo /foo/bar`, `negativeTreeWithPositiveBase = { bar = "directory"; }` + # because under the base path of `/foo`, only `bar` from the negative file set is included + # For `difference /foo/bar /foo`, `negativeTreeWithPositiveBase = "directory"` + # because under the base path of `/foo/bar`, everything from the negative file set is included + # For `difference /foo /bar`, `negativeTreeWithPositiveBase = null` + # because under the base path of `/foo`, nothing from the negative file set is included + negativeTreeWithPositiveBase = + if commonBaseComponentsLength == length positive._internalBaseComponents then + # The common prefix is the same as the positive base path, so the second path is equal or longer. + # We need to _shorten_ the negative filesetTree to the same base path as the positive one + # E.g. for `difference /foo /foo/bar` the common prefix is /foo, equal to the positive file set's base + # So we need to shorten the base of the tree for the negative argument from /foo/bar to just /foo + _shortenTreeBase positive._internalBaseComponents negative + else if commonBaseComponentsLength == length negative._internalBaseComponents then + # The common prefix is the same as the negative base path, so the first path is longer. + # We need to lengthen the negative filesetTree to the same base path as the positive one. + # E.g. for `difference /foo/bar /foo` the common prefix is /foo, equal to the negative file set's base + # So we need to lengthen the base of the tree for the negative argument from /foo to /foo/bar + _lengthenTreeBase positive._internalBaseComponents negative + else + # The common prefix is neither the first nor the second path. + # This means there's no overlap between the two file sets, + # and nothing from the negative argument should get removed from the positive one + # E.g for `difference /foo /bar`, we remove nothing to get the same as `/foo` + null; + + resultingTree = + _differenceTree + positive._internalBase + positive._internalTree + negativeTreeWithPositiveBase; + in + # If the first file set is empty, we can never have any files in the result + if positive._internalIsEmptyWithoutBase then + _emptyWithoutBase + # If the second file set is empty, nothing gets removed, so the result is just the first file set + else if negative._internalIsEmptyWithoutBase then + positive + else + # We use the positive file set base for the result, + # because only files from the positive side may be included, + # which is what base path is for + _create positive._internalBase resultingTree; + + # Computes the set difference of two filesetTree's + # Type: Path -> filesetTree -> filesetTree + _differenceTree = path: lhs: rhs: + # If the lhs doesn't have any files, or the right hand side includes all files + if lhs == null || isString rhs then + # The result will always be empty + null + # If the right hand side has no files + else if rhs == null then + # The result is always the left hand side, because nothing gets removed + lhs + else + # Otherwise we always have two attribute sets to recurse into + mapAttrs (name: lhsValue: + _differenceTree (path + "/${name}") lhsValue (rhs.${name} or null) + ) (_directoryEntries path lhs); + _fileFilter = predicate: fileset: let recurse = path: tree: diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index d8d8dd41318..d6ebdb098b5 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -677,6 +677,104 @@ tree=( ) checkFileset 'intersection (unions [ ./a/b ./c/d ./c/e ]) (unions [ ./a ./c/d/f ./c/e ])' +## Difference + +# Subtracting something from itself results in nothing +tree=( + [a]=0 +) +checkFileset 'difference ./. ./.' + +# The tree of the second argument should not be evaluated if not needed +checkFileset 'difference _emptyWithoutBase (_create ./. (abort "This should not be used!"))' +checkFileset 'difference (_create ./. null) (_create ./. (abort "This should not be used!"))' + +# Subtracting nothing gives the same thing back +tree=( + [a]=1 +) +checkFileset 'difference ./. _emptyWithoutBase' +checkFileset 'difference ./. (_create ./. null)' + +# Subtracting doesn't influence the base path +mkdir a b +touch {a,b}/x +expectEqual 'toSource { root = ./a; fileset = difference ./a ./b; }' 'toSource { root = ./a; fileset = ./a; }' +rm -rf -- * + +# Also not the other way around +mkdir a +expectFailure 'toSource { root = ./a; fileset = difference ./. ./a; }' 'lib.fileset.toSource: `fileset` could contain files in '"$work"', which is not under the `root` \('"$work"'/a\). Potential solutions: +\s*- Set `root` to '"$work"' or any directory higher up. This changes the layout of the resulting store path. +\s*- Set `fileset` to a file set that cannot contain files outside the `root` \('"$work"'/a\). This could change the files included in the result.' +rm -rf -- * + +# Difference actually works +# We test all combinations of ./., ./a, ./a/x and ./b +tree=( + [a/x]=0 + [a/y]=0 + [b]=0 + [c]=0 +) +checkFileset 'difference ./. ./.' +checkFileset 'difference ./a ./.' +checkFileset 'difference ./a/x ./.' +checkFileset 'difference ./b ./.' +checkFileset 'difference ./a ./a' +checkFileset 'difference ./a/x ./a' +checkFileset 'difference ./a/x ./a/x' +checkFileset 'difference ./b ./b' +tree=( + [a/x]=0 + [a/y]=0 + [b]=1 + [c]=1 +) +checkFileset 'difference ./. ./a' +tree=( + [a/x]=1 + [a/y]=1 + [b]=0 + [c]=0 +) +checkFileset 'difference ./a ./b' +tree=( + [a/x]=1 + [a/y]=0 + [b]=0 + [c]=0 +) +checkFileset 'difference ./a/x ./b' +tree=( + [a/x]=0 + [a/y]=1 + [b]=0 + [c]=0 +) +checkFileset 'difference ./a ./a/x' +tree=( + [a/x]=0 + [a/y]=0 + [b]=1 + [c]=0 +) +checkFileset 'difference ./b ./a' +checkFileset 'difference ./b ./a/x' +tree=( + [a/x]=0 + [a/y]=1 + [b]=1 + [c]=1 +) +checkFileset 'difference ./. ./a/x' +tree=( + [a/x]=1 + [a/y]=1 + [b]=0 + [c]=1 +) +checkFileset 'difference ./. ./b' ## File filter -- cgit 1.4.1 From b7e69faf1d256ce5a661e2cbab6a72f075d87f21 Mon Sep 17 00:00:00 2001 From: Alexander Groleau Date: Tue, 31 Oct 2023 11:45:51 -0700 Subject: lib/fixed-points.nix: correct typo Corrects a minor typo in the description of `fix`; namely that `fix` allows factoring out syntactic recursion from another function. --- lib/fixed-points.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix index 3444e95e15a..3b5fdc9e8ea 100644 --- a/lib/fixed-points.nix +++ b/lib/fixed-points.nix @@ -45,7 +45,7 @@ rec { } ``` - This is where `fix` comes in, it contains the syntactic that's not in `f` anymore. + This is where `fix` comes in, it contains the syntactic recursion that's not in `f` anymore. ```nix nix-repl> fix = f: -- cgit 1.4.1 From 206d20426cb2959aab8cc3cfc8e62aa68bc7b52f Mon Sep 17 00:00:00 2001 From: h7x4 Date: Tue, 17 Oct 2023 01:12:49 +0200 Subject: lib.strings: add `replicate` `strings.replicate` returns n copies of a string, concatenated into a new string Co-authored-by: Silvan Mosberger --- lib/strings.nix | 14 ++++++++++++++ lib/tests/misc.nix | 5 +++++ 2 files changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/strings.nix b/lib/strings.nix index 628669d86bb..695aaaacd34 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -144,6 +144,20 @@ rec { */ concatLines = concatMapStrings (s: s + "\n"); + /* + Replicate a string n times, + and concatenate the parts into a new string. + + Type: replicate :: int -> string -> string + + Example: + replicate 3 "v" + => "vvv" + replicate 5 "hello" + => "hellohellohellohellohello" + */ + replicate = n: s: concatStrings (lib.lists.replicate n s); + /* Construct a Unix-style, colon-separated search path consisting of the given `subDir` appended to each of the given paths. diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 2e7fda2b1f8..1fb857a65e9 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -191,6 +191,11 @@ runTests { expected = "a\nb\nc\n"; }; + testReplicateString = { + expr = strings.replicate 5 "hello"; + expected = "hellohellohellohellohello"; + }; + testSplitStringsSimple = { expr = strings.splitString "." "a.b.c.d"; expected = [ "a" "b" "c" "d" ]; -- cgit 1.4.1 From add254658af2e2904af0ee644a1b44b89f6c973a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 1 Nov 2023 18:56:03 +0100 Subject: lib.filesystem: Don't test Nix-specific error messages In https://github.com/NixOS/nix/pull/9269 the error messages change which would've broken this test. --- lib/tests/filesystem.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/tests/filesystem.sh b/lib/tests/filesystem.sh index cfd333d0001..7e7e03bc667 100755 --- a/lib/tests/filesystem.sh +++ b/lib/tests/filesystem.sh @@ -64,8 +64,14 @@ expectSuccess "pathType $PWD/directory" '"directory"' expectSuccess "pathType $PWD/regular" '"regular"' expectSuccess "pathType $PWD/symlink" '"symlink"' expectSuccess "pathType $PWD/fifo" '"unknown"' -# Different errors depending on whether the builtins.readFilePath primop is available or not -expectFailure "pathType $PWD/non-existent" "error: (evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'|getting status of '$PWD/non-existent': No such file or directory)" + +# Only check error message when a Nixpkgs-specified error is thrown, +# which is only the case when `readFileType` is not available +# and the fallback implementation needs to be used. +if [[ "$(nix-instantiate --eval --expr 'builtins ? readFileType')" == false ]]; then + expectFailure "pathType $PWD/non-existent" \ + "error: evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'" +fi expectSuccess "pathIsDirectory /." "true" expectSuccess "pathIsDirectory $PWD/directory" "true" -- cgit 1.4.1 From 2312523ab389cf6e5800ca7b9f7a368ccb964b97 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 1 Nov 2023 22:38:48 +0100 Subject: lib.fileset: Add an additional argument in the design docs --- lib/fileset/README.md | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index ebe13f08fde..16ab58e2f26 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -225,6 +225,9 @@ Arguments: This use case makes little sense for files that are already in the store. This should be a separate abstraction as e.g. `pkgs.drvLayout` instead, which could have a similar interface but be specific to derivations. Additional capabilities could be supported that can't be done at evaluation time, such as renaming files, creating new directories, setting executable bits, etc. +- (+) An API for filtering/transforming Nix store paths could be much more powerful, + because it's not limited to just what is possible at evaluation time with `builtins.path`. + Operations such as moving and adding files would be supported. ### Single files -- cgit 1.4.1 From dd48cb0b3479912a9c6adfaa7279dbadabaeae2b Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 2 Nov 2023 17:47:02 +0200 Subject: lib.makeScopeWithSplicing': add comments I didn't add these arguments, so these comments are from my understading of the arguments. --- lib/customisation.nix | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'lib') diff --git a/lib/customisation.nix b/lib/customisation.nix index 61bb531d2f6..c7d40339d05 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -343,7 +343,24 @@ rec { , newScope }: { otherSplices + # Attrs from `self` which won't be spliced. + # Avoid using keep, it's only used for a python hook workaround, added in PR #104201. + # ex: `keep = (self: { inherit (self) aAttr; })` , keep ? (_self: {}) + # Additional attrs to add to the sets `callPackage`. + # When the package is from a subset (but not a subset within a package IS #211340) + # within `spliced0` it will be spliced. + # When using an package outside the set but it's available from `pkgs`, use the package from `pkgs.__splicedPackages`. + # If the package is not available within the set or in `pkgs`, such as a package in a let binding, it will not be spliced + # ex: + # ``` + # nix-repl> darwin.apple_sdk.frameworks.CoreFoundation + # «derivation ...CoreFoundation-11.0.0.drv» + # nix-repl> darwin.CoreFoundation + # error: attribute 'CoreFoundation' missing + # nix-repl> darwin.callPackage ({ CoreFoundation }: CoreFoundation) { } + # «derivation ...CoreFoundation-11.0.0.drv» + # ``` , extra ? (_spliced0: {}) , f }: -- cgit 1.4.1 From 72c279f477f42f66cf7013d5a01762d32d215bff Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 2 Nov 2023 03:40:19 -0700 Subject: lib.systems, test.cross.sanity: add test case for #264989 We have several cross-compilation bugs that show up if hostPlatform!=buildPlatform yet hostPlatform.config==buildPlatform.config. These bugs have appeared and disappeared as we've fiddled with the definition of equality for platform objects. This commit adds a clear-cut case where they are *not* equal and never will be, so we can test it. --- lib/systems/examples.nix | 1 + pkgs/test/cross/default.nix | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/systems/examples.nix b/lib/systems/examples.nix index 0e704b7d7de..75578b97494 100644 --- a/lib/systems/examples.nix +++ b/lib/systems/examples.nix @@ -115,6 +115,7 @@ rec { }; gnu64 = { config = "x86_64-unknown-linux-gnu"; }; + gnu64_simplekernel = gnu64 // platforms.pc_simplekernel; # see test/cross/default.nix gnu32 = { config = "i686-unknown-linux-gnu"; }; musl64 = { config = "x86_64-unknown-linux-musl"; }; diff --git a/pkgs/test/cross/default.nix b/pkgs/test/cross/default.nix index f844fc2b8ec..124f6eec10a 100644 --- a/pkgs/test/cross/default.nix +++ b/pkgs/test/cross/default.nix @@ -134,6 +134,7 @@ let pkgs.pkgsMusl.stdenv pkgs.pkgsLLVM.stdenv pkgs.pkgsStatic.bash + #pkgs.pkgsCross.gnu64_simplekernel.bash # https://github.com/NixOS/nixpkgs/issues/264989 pkgs.pkgsCross.arm-embedded.stdenv pkgs.pkgsCross.sheevaplug.stdenv # for armv5tel pkgs.pkgsCross.raspberryPi.stdenv # for armv6l -- cgit 1.4.1 From 8c0416ef364f5d5dea9d5694aadc0014bb95c1ca Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Mon, 21 Aug 2023 14:38:53 -0700 Subject: lib.systems.inspect: add patternLogicalAnd --- lib/systems/inspect.nix | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'lib') diff --git a/lib/systems/inspect.nix b/lib/systems/inspect.nix index 022e459c394..073df78797c 100644 --- a/lib/systems/inspect.nix +++ b/lib/systems/inspect.nix @@ -100,6 +100,32 @@ rec { ]; }; + # given two patterns, return a pattern which is their logical AND. + # Since a pattern is a list-of-disjuncts, this needs to + patternLogicalAnd = pat1_: pat2_: + let + # patterns can be either a list or a (bare) singleton; turn + # them into singletons for uniform handling + pat1 = lib.toList pat1_; + pat2 = lib.toList pat2_; + in + lib.concatMap (attr1: + map (attr2: + lib.recursiveUpdateUntil + (path: subattr1: subattr2: + if (builtins.intersectAttrs subattr1 subattr2) == {} || subattr1 == subattr2 + then true + else throw '' + pattern conflict at path ${toString path}: + ${builtins.toJSON subattr1} + ${builtins.toJSON subattr2} + '') + attr1 + attr2 + ) + pat2) + pat1; + matchAnyAttrs = patterns: if builtins.isList patterns then attrs: any (pattern: matchAttrs pattern attrs) patterns else matchAttrs patterns; -- cgit 1.4.1 From a16319908e7bda2a9744114396e5e32a4ff6a3fc Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 29 May 2023 05:51:52 +0800 Subject: lib.mirrorFunctionArgs: init Co-authored-by: Silvan Mosberger --- lib/default.nix | 2 +- lib/trivial.nix | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/default.nix b/lib/default.nix index fe737a125e6..0dac50a08ca 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -74,7 +74,7 @@ let importJSON importTOML warn warnIf warnIfNot throwIf throwIfNot checkListOfEnum info showWarnings nixpkgsVersion version isInOldestRelease mod compare splitByAndCompare - functionArgs setFunctionArgs isFunction toFunction + functionArgs setFunctionArgs isFunction toFunction mirrorFunctionArgs toHexString toBaseDigits inPureEvalMode; inherit (self.fixedPoints) fix fix' converge extends composeExtensions composeManyExtensions makeExtensible makeExtensibleWithCustomName; diff --git a/lib/trivial.nix b/lib/trivial.nix index c23fc6070be..a89c1aa25b1 100644 --- a/lib/trivial.nix +++ b/lib/trivial.nix @@ -448,6 +448,40 @@ rec { isFunction = f: builtins.isFunction f || (f ? __functor && isFunction (f.__functor f)); + /* + `mirrorFunctionArgs f g` creates a new function `g'` with the same behavior as `g` (`g' x == g x`) + but its function arguments mirroring `f` (`lib.functionArgs g' == lib.functionArgs f`). + + Type: + mirrorFunctionArgs :: (a -> b) -> (a -> c) -> (a -> c) + + Example: + addab = {a, b}: a + b + addab { a = 2; b = 4; } + => 6 + lib.functionArgs addab + => { a = false; b = false; } + addab1 = attrs: addab attrs + 1 + addab1 { a = 2; b = 4; } + => 7 + lib.functionArgs addab1 + => { } + addab1' = lib.mirrorFunctionArgs addab addab1 + addab1' { a = 2; b = 4; } + => 7 + lib.functionArgs addab1' + => { a = false; b = false; } + */ + mirrorFunctionArgs = + # Function to provide the argument metadata + f: + let + fArgs = functionArgs f; + in + # Function to set the argument metadata to + g: + setFunctionArgs g fArgs; + /* Turns any non-callable values into constant functions. Returns callable values as is. -- cgit 1.4.1 From 6a8b6b8f720b8d6f43ea870164eb489de5316077 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 29 May 2023 05:52:21 +0800 Subject: lib.makeOverridable: simplify function arguments preservation Rename temporary variable copyArgs -> mirrorArgs. Use lib.mirrorFunctionArgs to define `mirrorArgs`. Apply mirrorArgs also to the returned function. --- lib/customisation.nix | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/customisation.nix b/lib/customisation.nix index c7d40339d05..08fc5db0614 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -76,19 +76,22 @@ rec { Type: makeOverridable :: (AttrSet -> a) -> AttrSet -> a */ - makeOverridable = f: lib.setFunctionArgs - (origArgs: let + makeOverridable = f: + let + # Creates a functor with the same arguments as f + mirrorArgs = lib.mirrorFunctionArgs f; + in + mirrorArgs (origArgs: + let result = f origArgs; - # Creates a functor with the same arguments as f - copyArgs = g: lib.setFunctionArgs g (lib.functionArgs f); # Changes the original arguments with (potentially a function that returns) a set of new attributes overrideWith = newArgs: origArgs // (if lib.isFunction newArgs then newArgs origArgs else newArgs); # Re-call the function but with different arguments - overrideArgs = copyArgs (newArgs: makeOverridable f (overrideWith newArgs)); + overrideArgs = mirrorArgs (newArgs: makeOverridable f (overrideWith newArgs)); # Change the result of the function call by applying g to it - overrideResult = g: makeOverridable (copyArgs (args: g (f args))) origArgs; + overrideResult = g: makeOverridable (mirrorArgs (args: g (f args))) origArgs; in if builtins.isAttrs result then result // { @@ -102,8 +105,7 @@ rec { lib.setFunctionArgs result (lib.functionArgs result) // { override = overrideArgs; } - else result) - (lib.functionArgs f); + else result); /* Call the package function in the file `fn` with the required -- cgit 1.4.1 From f82a1686e6840c396a141ff122e2f39d64fb6d4b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 8 Nov 2023 05:16:01 +0100 Subject: lib.fileset: Split out internal test helper --- lib/fileset/tests.sh | 68 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 529f23ae887..ead6d4c4956 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -224,6 +224,43 @@ withFileMonitor() { fi } + +# Create the tree structure declared in the tree variable, usage: +# +# tree=( +# [a/b] = # Declare that file a/b should exist +# [c/a] = # Declare that file c/a should exist +# [c/d/]= # Declare that directory c/d/ should exist +# ) +# createTree +declare -A tree +createTree() { + # Track which paths need to be created + local -a dirsToCreate=() + local -a filesToCreate=() + for p in "${!tree[@]}"; do + # If keys end with a `/` we treat them as directories, otherwise files + if [[ "$p" =~ /$ ]]; then + dirsToCreate+=("$p") + else + filesToCreate+=("$p") + fi + done + + # Create all the necessary paths. + # This is done with only a fixed number of processes, + # in order to not be too slow + # Though this does mean we're a bit limited with how many files can be created + if (( ${#dirsToCreate[@]} != 0 )); then + mkdir -p "${dirsToCreate[@]}" + fi + if (( ${#filesToCreate[@]} != 0 )); then + readarray -d '' -t parentsToCreate < <(dirname -z "${filesToCreate[@]}") + mkdir -p "${parentsToCreate[@]}" + touch "${filesToCreate[@]}" + fi +} + # Check whether a file set includes/excludes declared paths as expected, usage: # # tree=( @@ -232,34 +269,26 @@ withFileMonitor() { # [c/d/]= # Declare that directory c/d/ should exist and expect it to be excluded in the store path # ) # checkFileset './a' # Pass the fileset as the argument -declare -A tree checkFileset() { # New subshell so that we can have a separate trap handler, see `trap` below local fileset=$1 + # Create the tree + createTree + # Process the tree into separate arrays for included paths, excluded paths and excluded files. local -a included=() local -a excluded=() local -a excludedFiles=() - # Track which paths need to be created - local -a dirsToCreate=() - local -a filesToCreate=() for p in "${!tree[@]}"; do - # If keys end with a `/` we treat them as directories, otherwise files - if [[ "$p" =~ /$ ]]; then - dirsToCreate+=("$p") - isFile= - else - filesToCreate+=("$p") - isFile=1 - fi case "${tree[$p]}" in 1) included+=("$p") ;; 0) excluded+=("$p") - if [[ -n "$isFile" ]]; then + # If keys end with a `/` we treat them as directories, otherwise files + if [[ ! "$p" =~ /$ ]]; then excludedFiles+=("$p") fi ;; @@ -268,19 +297,6 @@ checkFileset() { esac done - # Create all the necessary paths. - # This is done with only a fixed number of processes, - # in order to not be too slow - # Though this does mean we're a bit limited with how many files can be created - if (( ${#dirsToCreate[@]} != 0 )); then - mkdir -p "${dirsToCreate[@]}" - fi - if (( ${#filesToCreate[@]} != 0 )); then - readarray -d '' -t parentsToCreate < <(dirname -z "${filesToCreate[@]}") - mkdir -p "${parentsToCreate[@]}" - touch "${filesToCreate[@]}" - fi - expression="toSource { root = ./.; fileset = $fileset; }" # We don't have lambda's in bash unfortunately, -- cgit 1.4.1 From 73493584a74cddb22d785942a2215c3dab12c29e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 18 Oct 2023 00:16:50 +0200 Subject: lib.fileset.fromSource: init --- lib/fileset/default.nix | 71 ++++++++++++++++ lib/fileset/internal.nix | 53 ++++++++++++ lib/fileset/tests.sh | 213 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 337 insertions(+) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 7bd70167038..53edc9e7709 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -3,8 +3,10 @@ let inherit (import ./internal.nix { inherit lib; }) _coerce + _singleton _coerceMany _toSourceFilter + _fromSourceFilter _unionMany _printFileset _intersection @@ -187,6 +189,75 @@ If a directory does not recursively contain any file, it is omitted from the sto filter = sourceFilter; }; + /* + Create a file set with the same files as a `lib.sources`-based value. + This does not import any of the files into the store. + + This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`. + + A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource). + + :::{.note} + File sets cannot represent empty directories. + Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories. + ::: + + Type: + fromSource :: SourceLike -> FileSet + + Example: + # There's no cleanSource-like function for file sets yet, + # but we can just convert cleanSource to a file set and use it that way + toSource { + root = ./.; + fileset = fromSource (lib.sources.cleanSource ./.); + } + + # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`), + # but removing a subdirectory using file set functions + difference + (fromSource (lib.sources.sourceByRegex ./. [ + "^README\.md$" + # This regex includes everything in ./doc + "^doc(/.*)?$" + ]) + ./doc/generated + + # Use cleanSource, but limit it to only include ./Makefile and files under ./src + intersection + (fromSource (lib.sources.cleanSource ./.)) + (unions [ + ./Makefile + ./src + ]); + */ + fromSource = source: + let + # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`, + # which are technically internal to lib.sources, + # but we'll allow this since both libraries are in the same code base + # and this function is a bridge between them. + isFiltered = source ? _isLibCleanSourceWith; + path = if isFiltered then source.origSrc else source; + in + # We can only support sources created from paths + if ! isPath path then + if isStringLike path then + throw '' + lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead. + Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'' + else + throw '' + lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fromSource: The source origin (${toString path}) of the argument does not exist.'' + else if isFiltered then + _fromSourceFilter path source.filter + else + # If there's no filter, no need to run the expensive conversion, all subpaths will be included + _singleton path; + /* The file set containing all files that are in either of two given file sets. This is the same as [`unions`](#function-library-lib.fileset.unions), diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 9892172955c..03126252078 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -461,6 +461,59 @@ rec { else nonEmpty; + # Turn a builtins.filterSource-based source filter on a root path into a file set + # containing only files included by the filter. + # The filter is lazily called as necessary to determine whether paths are included + # Type: Path -> (String -> String -> Bool) -> fileset + _fromSourceFilter = root: sourceFilter: + let + # During the recursion we need to track both: + # - The path value such that we can safely call `readDir` on it + # - The path string value such that we can correctly call the `filter` with it + # + # While we could just recurse with the path value, + # this would then require converting it to a path string for every path, + # which is a fairly expensive operation + + # Create a file set from a directory entry + fromDirEntry = path: pathString: type: + # The filter needs to run on the path as a string + if ! sourceFilter pathString type then + null + else if type == "directory" then + fromDir path pathString + else + type; + + # Create a file set from a directory + fromDir = path: pathString: + mapAttrs + # This looks a bit funny, but we need both the path-based and the path string-based values + (name: fromDirEntry (path + "/${name}") (pathString + "/${name}")) + # We need to readDir on the path value, because reading on a path string + # would be unspecified if there are multiple filesystem roots + (readDir path); + + rootPathType = pathType root; + + # We need to convert the path to a string to imitate what builtins.path calls the filter function with. + # We don't want to rely on `toString` for this though because it's not very well defined, see ../path/README.md + # So instead we use `lib.path.splitRoot` to safely deconstruct the path into its filesystem root and subpath + # We don't need the filesystem root though, builtins.path doesn't expose that in any way to the filter. + # So we only need the components, which we then turn into a string as one would expect. + rootString = "/" + concatStringsSep "/" (components (splitRoot root).subpath); + in + if rootPathType == "directory" then + # We imitate builtins.path not calling the filter on the root path + _create root (fromDir root rootString) + else + # Direct files are always included by builtins.path without calling the filter + # But we need to lift up the base path to its parent to satisfy the base path invariant + _create (dirOf root) + { + ${baseNameOf root} = rootPathType; + }; + # Transforms the filesetTree of a file set to a shorter base path, e.g. # _shortenTreeBase [ "foo" ] (_create /foo/bar null) # => { bar = null; } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index ead6d4c4956..13f02bb656b 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # shellcheck disable=SC2016 +# shellcheck disable=SC2317 +# shellcheck disable=SC2192 # Tests lib.fileset # Run: @@ -839,6 +841,217 @@ touch 0 "${filesToCreate[@]}" expectTrace 'unions (mapAttrsToList (n: _: ./. + "/${n}") (removeAttrs (builtins.readDir ./.) [ "0" ]))' "$expectedTrace" rm -rf -- * +## lib.fileset.fromSource + +# Check error messages +expectFailure 'fromSource null' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.' + +expectFailure 'fromSource (lib.cleanSource "")' 'lib.fileset.fromSource: The source origin of the argument is a string-like value \(""\), but it should be a path instead. +\s*Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.' + +expectFailure 'fromSource (lib.cleanSource null)' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.' + +# fromSource on a path works and is the same as coercing that path +mkdir a +touch a/b c +expectEqual 'trace (fromSource ./.) null' 'trace ./. null' +rm -rf -- * + +# Check that converting to a file set doesn't read the included files +mkdir a +touch a/b +run() { + expectEqual "trace (fromSource (lib.cleanSourceWith { src = ./a; })) null" "builtins.trace \"$work/a (all files in directory)\" null" + rm a/b +} +withFileMonitor run a/b +rm -rf -- * + +# Check that converting to a file set doesn't read entries for directories that are filtered out +mkdir -p a/b +touch a/b/c +run() { + expectEqual "trace (fromSource (lib.cleanSourceWith { + src = ./a; + filter = pathString: type: false; + })) null" "builtins.trace \"(empty)\" null" + rm a/b/c + rmdir a/b +} +withFileMonitor run a/b +rm -rf -- * + +# The filter is not needed on empty directories +expectEqual 'trace (fromSource (lib.cleanSourceWith { + src = ./.; + filter = abort "filter should not be needed"; +})) null' 'trace _emptyWithoutBase null' + +# Single files also work +touch a b +expectEqual 'trace (fromSource (cleanSourceWith { src = ./a; })) null' 'trace ./a null' +rm -rf -- * + +# For a tree assigning each subpath true/false, +# check whether a source filter with those results includes the same files +# as a file set created using fromSource. Usage: +# +# tree=( +# [a]=1 # ./a is a file and the filter should return true for it +# [b/]=0 # ./b is a directory and the filter should return false for it +# ) +# checkSource +checkSource() { + createTree + + # Serialise the tree as JSON (there's only minimal savings with jq, + # and we don't need to handle escapes) + { + echo "{" + first=1 + for p in "${!tree[@]}"; do + if [[ -z "$first" ]]; then + echo "," + else + first= + fi + echo "\"$p\":" + case "${tree[$p]}" in + 1) + echo "true" + ;; + 0) + echo "false" + ;; + *) + die "Unsupported tree value: ${tree[$p]}" + esac + done + echo "}" + } > "$tmp/tree.json" + + # An expression to create a source value with a filter matching the tree + sourceExpr=' + let + tree = importJSON '"$tmp"'/tree.json; + in + cleanSourceWith { + src = ./.; + filter = + pathString: type: + let + stripped = removePrefix (toString ./. + "/") pathString; + key = stripped + optionalString (type == "directory") "/"; + in + tree.${key} or + (throw "tree key ${key} missing"); + } + ' + + filesetExpr=' + toSource { + root = ./.; + fileset = fromSource ('"$sourceExpr"'); + } + ' + + # Turn both into store paths + sourceStorePath=$(expectStorePath "$sourceExpr") + filesetStorePath=$(expectStorePath "$filesetExpr") + + # Loop through each path in the tree + while IFS= read -r -d $'\0' subpath; do + if [[ ! -e "$sourceStorePath"/"$subpath" ]]; then + # If it's not in the source store path, it's also not in the file set store path + if [[ -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr doesn't contain $subpath, but the corresponding store path $filesetStorePath created via fromSource does contain $subpath" + fi + elif [[ -z "$(find "$sourceStorePath"/"$subpath" -type f)" ]]; then + # If it's an empty directory in the source store path, it shouldn't be in the file set store path + if [[ -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr contains the path $subpath without any files, but the corresponding store path $filesetStorePath created via fromSource didn't omit it" + fi + else + # If it's non-empty directory or a file, it should be in the file set store path + if [[ ! -e "$filesetStorePath"/"$subpath" ]]; then + die "The store path $sourceStorePath created by $expr contains the non-empty path $subpath, but the corresponding store path $filesetStorePath created via fromSource doesn't include it" + fi + fi + done < <(find . -mindepth 1 -print0) + + rm -rf -- * +} + +# Check whether the filter is evaluated correctly +tree=( + [a]= + [b/]= + [b/c]= + [b/d]= + [e/]= + [e/e/]= +) +# We fill out the above tree values with all possible combinations of 0 and 1 +# Then check whether a filter based on those return values gets turned into the corresponding file set +for i in $(seq 0 $((2 ** ${#tree[@]} - 1 ))); do + for p in "${!tree[@]}"; do + tree[$p]=$(( i % 2 )) + (( i /= 2 )) || true + done + checkSource +done + +# The filter is called with the same arguments in the same order +mkdir a e +touch a/b a/c d e +expectEqual ' + trace (fromSource (cleanSourceWith { + src = ./.; + filter = pathString: type: builtins.trace "${pathString} ${toString type}" true; + })) null +' ' + builtins.seq (cleanSourceWith { + src = ./.; + filter = pathString: type: builtins.trace "${pathString} ${toString type}" true; + }).outPath + builtins.trace "'"$work"' (all files in directory)" + null +' +rm -rf -- * + +# Test that if a directory is not included, the filter isn't called on its contents +mkdir a b +touch a/c b/d +expectEqual 'trace (fromSource (cleanSourceWith { + src = ./.; + filter = pathString: type: + if pathString == toString ./a then + false + else if pathString == toString ./b then + true + else if pathString == toString ./b/d then + true + else + abort "This filter should not be called with path ${pathString}"; +})) null' 'trace (_create ./. { b = "directory"; }) null' +rm -rf -- * + +# The filter is called lazily: +# If a later say intersection removes a part of the tree, the filter won't run on it +mkdir a d +touch a/{b,c} d/e +expectEqual 'trace (intersection ./a (fromSource (lib.cleanSourceWith { + src = ./.; + filter = pathString: type: + if pathString == toString ./a || pathString == toString ./a/b then + true + else if pathString == toString ./a/c then + false + else + abort "filter should not be called on ${pathString}"; +}))) null' 'trace ./a/b null' +rm -rf -- * + # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets echo >&2 tests ok -- cgit 1.4.1 From e07e80e8419f35d596712ecfff57259f0230a33c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 8 Nov 2023 21:44:44 +0100 Subject: lib.fileset.toSource: Mention fromSource in errors --- lib/fileset/README.md | 1 - lib/fileset/default.nix | 7 ++++++- lib/fileset/internal.nix | 7 ++++++- lib/fileset/tests.sh | 7 +++++++ 4 files changed, 19 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index ebe13f08fde..d7398438826 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -241,5 +241,4 @@ Here's a list of places in the library that need to be updated in the future: - > The file set library is currently somewhat limited but is being expanded to include more functions over time. in [the manual](../../doc/functions/fileset.section.md) -- If/Once a function to convert `lib.sources` values into file sets exists, the `_coerce` and `toSource` functions should be updated to mention that function in the error when such a value is passed - If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 53edc9e7709..640d0b49d3e 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -153,7 +153,12 @@ If a directory does not recursively contain any file, it is omitted from the sto sourceFilter = _toSourceFilter fileset; in if ! isPath root then - if isStringLike root then + if root ? _isLibCleanSourceWith then + throw '' + lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead. + To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`. + Note that this only works for sources created from paths.'' + else if isStringLike root then throw '' lib.fileset.toSource: `root` ("${toString root}") is a string-like value, but it should be a path instead. Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 03126252078..45115d5a8a3 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -170,7 +170,12 @@ rec { else value else if ! isPath value then - if isStringLike value then + if value ? _isLibCleanSourceWith then + throw '' + ${context} is a `lib.sources`-based value, but it should be a file set or a path instead. + To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`. + Note that this only works for sources created from paths.'' + else if isStringLike value then throw '' ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead. Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 13f02bb656b..61779452482 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -339,6 +339,10 @@ checkFileset() { expectFailure 'toSource { root = "/nix/store/foobar"; fileset = ./.; }' 'lib.fileset.toSource: `root` \("/nix/store/foobar"\) is a string-like value, but it should be a path instead. \s*Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' +expectFailure 'toSource { root = cleanSourceWith { src = ./.; }; fileset = ./.; }' 'lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead. +\s*To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`. +\s*Note that this only works for sources created from paths.' + # Only paths are accepted as `root` expectFailure 'toSource { root = 10; fileset = ./.; }' 'lib.fileset.toSource: `root` is of type int, but it should be a path instead.' @@ -376,6 +380,9 @@ rm -rf * expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.' expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a file set or a path instead. \s*Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' +expectFailure 'toSource { root = ./.; fileset = cleanSourceWith { src = ./.; }; }' 'lib.fileset.toSource: `fileset` is a `lib.sources`-based value, but it should be a file set or a path instead. +\s*To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`. +\s*Note that this only works for sources created from paths.' # Path coercion errors for non-existent paths expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) does not exist.' -- cgit 1.4.1 From 3d1d5533f856e7831d60ad06094b135305f51e0f Mon Sep 17 00:00:00 2001 From: Tyler Benster Date: Mon, 23 Dec 2019 00:31:09 -0800 Subject: lib.licenses: add NVIDIA licenses --- lib/licenses.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index d9555ca66cb..e6005c3baef 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -877,6 +877,18 @@ in mkLicense lset) ({ fullName = "Non-Profit Open Software License 3.0"; }; + nvidiaCuda = { + fullName = "CUDA Toolkit Supplement to Software License Agreement for NVIDIA Software Development Kits"; + url = "https://docs.nvidia.com/cuda/eula/index.html#cuda-toolkit-supplement-license-agreement"; + free = false; + }; + + nvidiaCudnn = { + fullName = "cuDNN Supplement to Software License Agreement for NVIDIA Software Development Kits"; + url = "https://docs.nvidia.com/deeplearning/sdk/cudnn-sla/index.html#supplement"; + free = false; + }; + obsidian = { fullName = "Obsidian End User Agreement"; url = "https://obsidian.md/eula"; -- cgit 1.4.1 From 75254979ac1e4b7ff031b4c31fc121824df83743 Mon Sep 17 00:00:00 2001 From: Tyler Benster Date: Mon, 23 Dec 2019 00:33:19 -0800 Subject: cudnn_cudatoolkit: add NVIDIA license --- lib/licenses.nix | 6 ------ pkgs/development/libraries/science/math/cudnn/generic.nix | 9 +++++++-- 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index e6005c3baef..beb4be39108 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -883,12 +883,6 @@ in mkLicense lset) ({ free = false; }; - nvidiaCudnn = { - fullName = "cuDNN Supplement to Software License Agreement for NVIDIA Software Development Kits"; - url = "https://docs.nvidia.com/deeplearning/sdk/cudnn-sla/index.html#supplement"; - free = false; - }; - obsidian = { fullName = "Obsidian End User Agreement"; url = "https://obsidian.md/eula"; diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index b1e03e35df1..709974070e8 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -153,8 +153,13 @@ in description = "NVIDIA CUDA Deep Neural Network library (cuDNN)"; homepage = "https://developer.nvidia.com/cudnn"; sourceProvenance = with sourceTypes; [binaryNativeCode]; - # TODO: consider marking unfreRedistributable when not using runfile - license = licenses.unfree; + license = { + fullName = "cuDNN Supplement to Software License Agreement for NVIDIA Software Development Kits"; + url = "https://docs.nvidia.com/deeplearning/sdk/cudnn-sla/index.html#supplement"; + free = false; + } // lib.optionalAttrs (!useCudatoolkitRunfile) { + redistributable = true; + }; platforms = ["x86_64-linux"]; maintainers = with maintainers; [mdaiter samuela]; # Force the use of the default, fat output by default (even though `dev` exists, which -- cgit 1.4.1 From 6c9c9ef2ebb02e6b1d5482d93d6b18586034f8e8 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 17 Oct 2023 23:40:47 +0300 Subject: lib.licenses: nvidia*: use same fullNames as conda does --- lib/licenses.nix | 2 +- pkgs/development/libraries/science/math/cudnn/generic.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index beb4be39108..7160582b5a9 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -878,7 +878,7 @@ in mkLicense lset) ({ }; nvidiaCuda = { - fullName = "CUDA Toolkit Supplement to Software License Agreement for NVIDIA Software Development Kits"; + fullName = "CUDA Toolkit End User License Agreement (EULA)"; url = "https://docs.nvidia.com/cuda/eula/index.html#cuda-toolkit-supplement-license-agreement"; free = false; }; diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index 709974070e8..b34dbae336d 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -154,7 +154,7 @@ in homepage = "https://developer.nvidia.com/cudnn"; sourceProvenance = with sourceTypes; [binaryNativeCode]; license = { - fullName = "cuDNN Supplement to Software License Agreement for NVIDIA Software Development Kits"; + fullName = "NVIDIA cuDNN Software License Agreement (EULA)"; url = "https://docs.nvidia.com/deeplearning/sdk/cudnn-sla/index.html#supplement"; free = false; } // lib.optionalAttrs (!useCudatoolkitRunfile) { -- cgit 1.4.1 From 6cd3e9ce1a638711166fdd2a4bb3398ca2c9e081 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 17 Oct 2023 23:44:10 +0300 Subject: cudaPackages: redist components: per-package license url --- lib/licenses.nix | 8 ++++++++ .../compilers/cudatoolkit/redist/build-cuda-redist-package.nix | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index 7160582b5a9..f8f596a66cd 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -883,6 +883,14 @@ in mkLicense lset) ({ free = false; }; + nvidiaCudaRedist = { + shortName = "CUDA EULA"; + fullName = "CUDA Toolkit End User License Agreement (EULA)"; + url = "https://docs.nvidia.com/cuda/eula/index.html#cuda-toolkit-supplement-license-agreement"; + free = false; + redistributable = true; + }; + obsidian = { fullName = "Obsidian End User Agreement"; url = "https://obsidian.md/eula"; diff --git a/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix b/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix index 038841a7097..85c3182c052 100644 --- a/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix +++ b/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix @@ -165,7 +165,9 @@ backendStdenv.mkDerivation { meta = { inherit description platforms; - license = lib.licenses.unfree; + license = lib.licenses.nvidiaCudaRedist // { + url = "https://developer.download.nvidia.com/compute/cuda/redist/${releaseAttrs.license_path or "${pname}/LICENSE.txt"}"; + }; maintainers = lib.teams.cuda.members; # Force the use of the default, fat output by default (even though `dev` exists, which # causes Nix to prefer that output over the others if outputSpecified isn't set). -- cgit 1.4.1 From dd8e4b0a02dfca89e5de20aff6dfbebe5a7d136b Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 17 Oct 2023 23:57:02 +0300 Subject: lib.licenses: nvidia*: add shortName-s --- lib/licenses.nix | 1 + pkgs/development/libraries/science/math/cudnn/generic.nix | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/licenses.nix b/lib/licenses.nix index f8f596a66cd..a90bab0b15d 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -878,6 +878,7 @@ in mkLicense lset) ({ }; nvidiaCuda = { + shortName = "CUDA EULA"; fullName = "CUDA Toolkit End User License Agreement (EULA)"; url = "https://docs.nvidia.com/cuda/eula/index.html#cuda-toolkit-supplement-license-agreement"; free = false; diff --git a/pkgs/development/libraries/science/math/cudnn/generic.nix b/pkgs/development/libraries/science/math/cudnn/generic.nix index b34dbae336d..b9f101d80fa 100644 --- a/pkgs/development/libraries/science/math/cudnn/generic.nix +++ b/pkgs/development/libraries/science/math/cudnn/generic.nix @@ -154,6 +154,7 @@ in homepage = "https://developer.nvidia.com/cudnn"; sourceProvenance = with sourceTypes; [binaryNativeCode]; license = { + shortName = "cuDNN EULA"; fullName = "NVIDIA cuDNN Software License Agreement (EULA)"; url = "https://docs.nvidia.com/deeplearning/sdk/cudnn-sla/index.html#supplement"; free = false; -- cgit 1.4.1 From 2d28e0cd109dc9f04d2ab0d8ac90395feb2acb3e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 8 Nov 2023 23:28:25 +0100 Subject: lib.fileset: Remove nixdoc workaround This problem was fixed in https://github.com/nix-community/nixdoc/pull/81 which is included in version 2.5.1, which is now used in Nixpkgs --- lib/fileset/default.nix | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 4a97633b4a8..547809cf063 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -122,11 +122,10 @@ in { Paths in [strings](https://nixos.org/manual/nix/stable/language/values.html#type-string), including Nix store paths, cannot be passed as `root`. `root` has to be a directory. - -:::{.note} -Changing `root` only affects the directory structure of the resulting store path, it does not change which files are added to the store. -The only way to change which files get added to the store is by changing the `fileset` attribute. -::: + :::{.note} + Changing `root` only affects the directory structure of the resulting store path, it does not change which files are added to the store. + The only way to change which files get added to the store is by changing the `fileset` attribute. + ::: */ root, /* @@ -135,10 +134,9 @@ The only way to change which files get added to the store is by changing the `fi This argument can also be a path, which gets [implicitly coerced to a file set](#sec-fileset-path-coercion). - -:::{.note} -If a directory does not recursively contain any file, it is omitted from the store path contents. -::: + :::{.note} + If a directory does not recursively contain any file, it is omitted from the store path contents. + ::: */ fileset, -- cgit 1.4.1 From 0ace383438b84b0db04b4103d0b2d73400294058 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 8 Nov 2023 23:59:04 +0100 Subject: lib.fileset: Make error messages more uniform Just minor changes like: - Always using "X is a Y, but it should be Z" - "X is a path that does not exist" rather than "X does not exist" - Always using multi-line strings for errors - Always quoting string-like values and not quoting path-like values - But do quote filesystem roots. Even though they're paths, they might be very small, good to have quotes to know the start/end - Capitalise the first word - Distinguish root vs filesystem root more --- lib/fileset/default.nix | 38 ++++++++++++++++++++------------------ lib/fileset/internal.nix | 8 ++++---- lib/fileset/tests.sh | 46 +++++++++++++++++++++++----------------------- 3 files changed, 47 insertions(+), 45 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 547809cf063..fe7b304ba69 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -154,7 +154,7 @@ in { if ! isPath root then if isStringLike root then throw '' - lib.fileset.toSource: `root` ("${toString root}") is a string-like value, but it should be a path instead. + lib.fileset.toSource: `root` (${toString root}) is a string-like value, but it should be a path instead. Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'' else throw '' @@ -163,13 +163,13 @@ in { # See also ../path/README.md else if ! fileset._internalIsEmptyWithoutBase && rootFilesystemRoot != filesetFilesystemRoot then throw '' - lib.fileset.toSource: Filesystem roots are not the same for `fileset` and `root` ("${toString root}"): - `root`: root "${toString rootFilesystemRoot}" - `fileset`: root "${toString filesetFilesystemRoot}" - Different roots are not supported.'' + lib.fileset.toSource: Filesystem roots are not the same for `fileset` and `root` (${toString root}): + `root`: Filesystem root is "${toString rootFilesystemRoot}" + `fileset`: Filesystem root is "${toString filesetFilesystemRoot}" + Different filesystem roots are not supported.'' else if ! pathExists root then throw '' - lib.fileset.toSource: `root` (${toString root}) does not exist.'' + lib.fileset.toSource: `root` (${toString root}) is a path that does not exist.'' else if pathType root != "directory" then throw '' lib.fileset.toSource: `root` (${toString root}) is a file, but it should be a directory instead. Potential solutions: @@ -221,11 +221,11 @@ in { _unionMany (_coerceMany "lib.fileset.union" [ { - context = "first argument"; + context = "First argument"; value = fileset1; } { - context = "second argument"; + context = "Second argument"; value = fileset2; } ]); @@ -267,12 +267,13 @@ in { # which get [implicitly coerced to file sets](#sec-fileset-path-coercion). filesets: if ! isList filesets then - throw "lib.fileset.unions: Expected argument to be a list, but got a ${typeOf filesets}." + throw '' + lib.fileset.unions: Argument is of type ${typeOf filesets}, but it should be a list instead.'' else pipe filesets [ # Annotate the elements with context, used by _coerceMany for better errors (imap0 (i: el: { - context = "element ${toString i}"; + context = "Element ${toString i}"; value = el; })) (_coerceMany "lib.fileset.unions") @@ -323,10 +324,11 @@ in { # The file set to filter based on the predicate function fileset: if ! isFunction predicate then - throw "lib.fileset.fileFilter: Expected the first argument to be a function, but it's a ${typeOf predicate} instead." + throw '' + lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function.'' else _fileFilter predicate - (_coerce "lib.fileset.fileFilter: second argument" fileset); + (_coerce "lib.fileset.fileFilter: Second argument" fileset); /* The file set containing all files that are in both of two given file sets. @@ -354,11 +356,11 @@ in { let filesets = _coerceMany "lib.fileset.intersection" [ { - context = "first argument"; + context = "First argument"; value = fileset1; } { - context = "second argument"; + context = "Second argument"; value = fileset2; } ]; @@ -406,11 +408,11 @@ in { let filesets = _coerceMany "lib.fileset.difference" [ { - context = "first argument (positive set)"; + context = "First argument (positive set)"; value = positive; } { - context = "second argument (negative set)"; + context = "Second argument (negative set)"; value = negative; } ]; @@ -454,7 +456,7 @@ in { let # "fileset" would be a better name, but that would clash with the argument name, # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.trace: argument" fileset; + actualFileset = _coerce "lib.fileset.trace: Argument" fileset; in seq (_printFileset actualFileset) @@ -501,7 +503,7 @@ in { let # "fileset" would be a better name, but that would clash with the argument name, # and we cannot change that because of https://github.com/nix-community/nixdoc/issues/76 - actualFileset = _coerce "lib.fileset.traceVal: argument" fileset; + actualFileset = _coerce "lib.fileset.traceVal: Argument" fileset; in seq (_printFileset actualFileset) diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index b919a5de3ee..853115df9f6 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -179,7 +179,7 @@ rec { ${context} is of type ${typeOf value}, but it should be a file set or a path instead.'' else if ! pathExists value then throw '' - ${context} (${toString value}) does not exist.'' + ${context} (${toString value}) is a path that does not exist.'' else _singleton value; @@ -208,9 +208,9 @@ rec { if firstWithBase != null && differentIndex != null then throw '' ${functionContext}: Filesystem roots are not the same: - ${(head list).context}: root "${toString firstBaseRoot}" - ${(elemAt list differentIndex).context}: root "${toString (elemAt filesets differentIndex)._internalBaseRoot}" - Different roots are not supported.'' + ${(head list).context}: Filesystem root is "${toString firstBaseRoot}" + ${(elemAt list differentIndex).context}: Filesystem root is "${toString (elemAt filesets differentIndex)._internalBaseRoot}" + Different filesystem roots are not supported.'' else filesets; diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 2df0727bde3..c1c67800f5e 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -318,7 +318,7 @@ checkFileset() { #### Error messages ##### # Absolute paths in strings cannot be passed as `root` -expectFailure 'toSource { root = "/nix/store/foobar"; fileset = ./.; }' 'lib.fileset.toSource: `root` \("/nix/store/foobar"\) is a string-like value, but it should be a path instead. +expectFailure 'toSource { root = "/nix/store/foobar"; fileset = ./.; }' 'lib.fileset.toSource: `root` \(/nix/store/foobar\) is a string-like value, but it should be a path instead. \s*Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' # Only paths are accepted as `root` @@ -328,14 +328,14 @@ expectFailure 'toSource { root = 10; fileset = ./.; }' 'lib.fileset.toSource: `r mkdir -p {foo,bar}/mock-root expectFailure 'with ((import ).extend (import )).fileset; toSource { root = ./foo/mock-root; fileset = ./bar/mock-root; } -' 'lib.fileset.toSource: Filesystem roots are not the same for `fileset` and `root` \("'"$work"'/foo/mock-root"\): -\s*`root`: root "'"$work"'/foo/mock-root" -\s*`fileset`: root "'"$work"'/bar/mock-root" -\s*Different roots are not supported.' +' 'lib.fileset.toSource: Filesystem roots are not the same for `fileset` and `root` \('"$work"'/foo/mock-root\): +\s*`root`: Filesystem root is "'"$work"'/foo/mock-root" +\s*`fileset`: Filesystem root is "'"$work"'/bar/mock-root" +\s*Different filesystem roots are not supported.' rm -rf -- * # `root` needs to exist -expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: `root` \('"$work"'/a\) does not exist.' +expectFailure 'toSource { root = ./a; fileset = ./.; }' 'lib.fileset.toSource: `root` \('"$work"'/a\) is a path that does not exist.' # `root` needs to be a file touch a @@ -367,7 +367,7 @@ expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.to \s*Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.' # Path coercion errors for non-existent paths -expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) does not exist.' +expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) is a path that does not exist.' # File sets cannot be evaluated directly expectFailure 'union ./. ./.' 'lib.fileset: Directly evaluating a file set is not supported. @@ -490,26 +490,26 @@ mkdir -p {foo,bar}/mock-root expectFailure 'with ((import ).extend (import )).fileset; toSource { root = ./.; fileset = union ./foo/mock-root ./bar/mock-root; } ' 'lib.fileset.union: Filesystem roots are not the same: -\s*first argument: root "'"$work"'/foo/mock-root" -\s*second argument: root "'"$work"'/bar/mock-root" -\s*Different roots are not supported.' +\s*First argument: Filesystem root is "'"$work"'/foo/mock-root" +\s*Second argument: Filesystem root is "'"$work"'/bar/mock-root" +\s*Different filesystem roots are not supported.' expectFailure 'with ((import ).extend (import )).fileset; toSource { root = ./.; fileset = unions [ ./foo/mock-root ./bar/mock-root ]; } ' 'lib.fileset.unions: Filesystem roots are not the same: -\s*element 0: root "'"$work"'/foo/mock-root" -\s*element 1: root "'"$work"'/bar/mock-root" -\s*Different roots are not supported.' +\s*Element 0: Filesystem root is "'"$work"'/foo/mock-root" +\s*Element 1: Filesystem root is "'"$work"'/bar/mock-root" +\s*Different filesystem roots are not supported.' rm -rf -- * # Coercion errors show the correct context -expectFailure 'toSource { root = ./.; fileset = union ./a ./.; }' 'lib.fileset.union: first argument \('"$work"'/a\) does not exist.' -expectFailure 'toSource { root = ./.; fileset = union ./. ./b; }' 'lib.fileset.union: second argument \('"$work"'/b\) does not exist.' -expectFailure 'toSource { root = ./.; fileset = unions [ ./a ./. ]; }' 'lib.fileset.unions: element 0 \('"$work"'/a\) does not exist.' -expectFailure 'toSource { root = ./.; fileset = unions [ ./. ./b ]; }' 'lib.fileset.unions: element 1 \('"$work"'/b\) does not exist.' +expectFailure 'toSource { root = ./.; fileset = union ./a ./.; }' 'lib.fileset.union: First argument \('"$work"'/a\) is a path that does not exist.' +expectFailure 'toSource { root = ./.; fileset = union ./. ./b; }' 'lib.fileset.union: Second argument \('"$work"'/b\) is a path that does not exist.' +expectFailure 'toSource { root = ./.; fileset = unions [ ./a ./. ]; }' 'lib.fileset.unions: Element 0 \('"$work"'/a\) is a path that does not exist.' +expectFailure 'toSource { root = ./.; fileset = unions [ ./. ./b ]; }' 'lib.fileset.unions: Element 1 \('"$work"'/b\) is a path that does not exist.' # unions needs a list -expectFailure 'toSource { root = ./.; fileset = unions null; }' 'lib.fileset.unions: Expected argument to be a list, but got a null.' +expectFailure 'toSource { root = ./.; fileset = unions null; }' 'lib.fileset.unions: Argument is of type null, but it should be a list instead.' # The tree of later arguments should not be evaluated if a former argument already includes all files tree=() @@ -603,14 +603,14 @@ mkdir -p {foo,bar}/mock-root expectFailure 'with ((import ).extend (import )).fileset; toSource { root = ./.; fileset = intersection ./foo/mock-root ./bar/mock-root; } ' 'lib.fileset.intersection: Filesystem roots are not the same: -\s*first argument: root "'"$work"'/foo/mock-root" -\s*second argument: root "'"$work"'/bar/mock-root" -\s*Different roots are not supported.' +\s*First argument: Filesystem root is "'"$work"'/foo/mock-root" +\s*Second argument: Filesystem root is "'"$work"'/bar/mock-root" +\s*Different filesystem roots are not supported.' rm -rf -- * # Coercion errors show the correct context -expectFailure 'toSource { root = ./.; fileset = intersection ./a ./.; }' 'lib.fileset.intersection: first argument \('"$work"'/a\) does not exist.' -expectFailure 'toSource { root = ./.; fileset = intersection ./. ./b; }' 'lib.fileset.intersection: second argument \('"$work"'/b\) does not exist.' +expectFailure 'toSource { root = ./.; fileset = intersection ./a ./.; }' 'lib.fileset.intersection: First argument \('"$work"'/a\) is a path that does not exist.' +expectFailure 'toSource { root = ./.; fileset = intersection ./. ./b; }' 'lib.fileset.intersection: Second argument \('"$work"'/b\) is a path that does not exist.' # The tree of later arguments should not be evaluated if a former argument already excludes all files tree=( -- cgit 1.4.1 From 9cbd394aa036442e0ccd0459908b96a5a60b2a9e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 9 Nov 2023 00:02:43 +0100 Subject: lib.fileset: Remove unused bindings Thanks nixd! --- lib/fileset/internal.nix | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 853115df9f6..d55c84a395e 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -7,7 +7,6 @@ let isString pathExists readDir - seq split trace typeOf @@ -17,7 +16,6 @@ let attrNames attrValues mapAttrs - setAttrByPath zipAttrsWith ; @@ -28,7 +26,6 @@ let inherit (lib.lists) all commonPrefix - drop elemAt filter findFirst -- cgit 1.4.1 From e3e57b8f1885bf1dc4787728479711f46a4171cc Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 9 May 2023 13:38:32 +0000 Subject: lib.systems: elaborate Rust metadata We need this stuff to be available in lib so make-derivation.nix can access it to construct the Meson cross file. This has a couple of other advantages: - It makes Rust less special. Now figuring out what Rust calls a platform is the same as figuring out what Linux or QEMU call it. - We can unify the schema used to define Rust targets, and the schema used to access those values later. Just like you can set "config" or "system" in a platform definition, and then access those same keys on the elaborated platform, you can now set "rustcTarget" in your crossSystem, and then access "stdenv.hostPlatform.rustcTarget" in your code. "rustcTarget", "rustcTargetSpec", "cargoShortTarget", and "cargoEnvVarTarget" have the "rustc" and "cargo" prefixes because these are not exposed to code by the compiler, and are not standardized. The arch/os/etc. variables are all named to match the forms in the Rust target spec JSON. The new rust.target-family only takes a list, since we don't need to worry about backwards compatibility when that name is used. The old APIs are all still functional with no warning for now, so that it's possible for external code to use a single API on both 23.05 and 23.11. We can introduce the warnings once 23.05 is EOL, and make them hard errors when 23.11 is EOL. --- lib/systems/default.nix | 100 +++++++++++++++- pkgs/applications/blockchains/zcash/default.nix | 4 +- pkgs/applications/misc/effitask/default.nix | 3 +- .../window-managers/cosmic/applets/default.nix | 6 +- .../window-managers/cosmic/panel/default.nix | 6 +- .../window-managers/cosmic/settings/default.nix | 2 +- .../rust/build-rust-crate/build-crate.nix | 3 +- .../rust/build-rust-crate/configure-crate.nix | 10 +- .../rust/build-rust-crate/default.nix | 11 +- .../rust/build-rust-package/default.nix | 5 +- .../rust/build-rust-package/sysroot/default.nix | 4 +- pkgs/build-support/rust/hooks/default.nix | 6 +- pkgs/build-support/rust/lib/default.nix | 127 +++++---------------- pkgs/desktops/gnome/core/gnome-tour/default.nix | 3 +- pkgs/development/compilers/mrustc/bootstrap.nix | 6 +- pkgs/development/compilers/rust/bootstrap.nix | 4 +- pkgs/development/compilers/rust/cargo.nix | 4 +- pkgs/development/compilers/rust/rustc.nix | 38 +++--- .../interpreters/python/hooks/default.nix | 8 +- .../development/libraries/gstreamer/rs/default.nix | 3 +- pkgs/development/libraries/libdovi/default.nix | 10 +- .../libraries/libimagequant/default.nix | 7 +- pkgs/development/libraries/librsvg/default.nix | 3 +- pkgs/development/libraries/relibc/default.nix | 2 +- .../libraries/rustc-demangle/default.nix | 4 +- pkgs/development/tools/lalrpop/default.nix | 3 +- .../tools/rust/cargo-benchcmp/default.nix | 3 +- .../development/tools/rust/cargo-watch/default.nix | 3 +- pkgs/development/web/deno/librusty_v8.nix | 5 +- pkgs/development/web/deno/update/librusty_v8.ts | 5 +- pkgs/development/web/edge-runtime/librusty_v8.nix | 5 +- pkgs/servers/http/router/librusty_v8.nix | 5 +- .../matrix-synapse/matrix-hookshot/default.nix | 3 +- pkgs/servers/windmill/default.nix | 4 +- pkgs/tools/misc/halp/default.nix | 3 +- pkgs/tools/system/zram-generator/default.nix | 3 +- pkgs/tools/video/rav1e/default.nix | 9 +- 37 files changed, 211 insertions(+), 219 deletions(-) (limited to 'lib') diff --git a/lib/systems/default.nix b/lib/systems/default.nix index 2790ea08d97..ada8c66e361 100644 --- a/lib/systems/default.nix +++ b/lib/systems/default.nix @@ -43,6 +43,10 @@ rec { elaborate = args': let args = if lib.isString args' then { system = args'; } else args'; + + # TODO: deprecate args.rustc in favour of args.rust after 23.05 is EOL. + rust = assert !(args ? rust && args ? rustc); args.rust or args.rustc or {}; + final = { # Prefer to parse `config` as it is strictly more informative. parsed = parse.mkSystemFromString (if args ? config then args.config else args.system); @@ -159,9 +163,101 @@ rec { ({ linux-kernel = args.linux-kernel or {}; gcc = args.gcc or {}; - rustc = args.rustc or {}; } // platforms.select final) - linux-kernel gcc rustc; + linux-kernel gcc; + + # TODO: remove after 23.05 is EOL, with an error pointing to the rust.* attrs. + rustc = args.rustc or {}; + + rust = rust // { + # Once args.rustc.platform.target-family is deprecated and + # removed, there will no longer be any need to modify any + # values from args.rust.platform, so we can drop all the + # "args ? rust" etc. checks, and merge args.rust.platform in + # /after/. + platform = rust.platform or {} // { + # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch + arch = + /**/ if rust ? platform then rust.platform.arch + else if final.isAarch32 then "arm" + else if final.isMips64 then "mips64" # never add "el" suffix + else if final.isPower64 then "powerpc64" # never add "le" suffix + else final.parsed.cpu.name; + + # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os + os = + /**/ if rust ? platform then rust.platform.os or "none" + else if final.isDarwin then "macos" + else final.parsed.kernel.name; + + # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family + target-family = + /**/ if args ? rust.platform.target-family then args.rust.platform.target-family + else if args ? rustc.platform.target-family + then + ( + # Since https://github.com/rust-lang/rust/pull/84072 + # `target-family` is a list instead of single value. + let + f = args.rustc.platform.target-family; + in + if builtins.isList f then f else [ f ] + ) + else lib.optional final.isUnix "unix" + ++ lib.optional final.isWindows "windows"; + + # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor + vendor = let + inherit (final.parsed) vendor; + in rust.platform.vendor or { + "w64" = "pc"; + }.${vendor.name} or vendor.name; + }; + + # The name of the rust target, even if it is custom. Adjustments are + # because rust has slightly different naming conventions than we do. + rustcTarget = let + inherit (final.parsed) cpu kernel abi; + cpu_ = rust.platform.arch or { + "armv7a" = "armv7"; + "armv7l" = "armv7"; + "armv6l" = "arm"; + "armv5tel" = "armv5te"; + "riscv64" = "riscv64gc"; + }.${cpu.name} or cpu.name; + vendor_ = final.rust.platform.vendor; + in rust.config + or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}"; + + # The name of the rust target if it is standard, or the json file + # containing the custom target spec. + rustcTargetSpec = + /**/ if rust ? platform + then builtins.toFile (final.rust.rustcTarget + ".json") (builtins.toJSON rust.platform) + else final.rust.rustcTarget; + + # The name of the rust target if it is standard, or the + # basename of the file containing the custom target spec, + # without the .json extension. + # + # This is the name used by Cargo for target subdirectories. + cargoShortTarget = + lib.removeSuffix ".json" (baseNameOf "${final.rust.rustcTargetSpec}"); + + # When used as part of an environment variable name, triples are + # uppercased and have all hyphens replaced by underscores: + # + # https://github.com/rust-lang/cargo/pull/9169 + # https://github.com/rust-lang/cargo/issues/8285#issuecomment-634202431 + cargoEnvVarTarget = + lib.strings.replaceStrings ["-"] ["_"] + (lib.strings.toUpper final.rust.cargoShortTarget); + + # True if the target is no_std + # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421 + isNoStdTarget = + builtins.any (t: lib.hasInfix t final.rust.rustcTarget) ["-none" "nvptx" "switch" "-uefi"]; + }; linuxArch = if final.isAarch32 then "arm" diff --git a/pkgs/applications/blockchains/zcash/default.nix b/pkgs/applications/blockchains/zcash/default.nix index c07f94e5bae..dbada09c1b6 100644 --- a/pkgs/applications/blockchains/zcash/default.nix +++ b/pkgs/applications/blockchains/zcash/default.nix @@ -1,5 +1,5 @@ { autoreconfHook, boost180, cargo, coreutils, curl, cxx-rs, db62, fetchFromGitHub -, git, hexdump, lib, libevent, libsodium, makeWrapper, rust, rustPlatform +, git, hexdump, lib, libevent, libsodium, makeWrapper, rustPlatform , pkg-config, Security, stdenv, testers, tl-expected, utf8cpp, util-linux, zcash, zeromq }: @@ -57,7 +57,7 @@ rustPlatform.buildRustPackage.override { inherit stdenv; } rec { configureFlags = [ "--disable-tests" "--with-boost-libdir=${lib.getLib boost180}/lib" - "RUST_TARGET=${rust.toRustTargetSpec stdenv.hostPlatform}" + "RUST_TARGET=${stdenv.hostPlatform.rust.rustcTargetSpec}" ]; enableParallelBuilding = true; diff --git a/pkgs/applications/misc/effitask/default.nix b/pkgs/applications/misc/effitask/default.nix index a380a06741e..50172b9d1cb 100644 --- a/pkgs/applications/misc/effitask/default.nix +++ b/pkgs/applications/misc/effitask/default.nix @@ -5,7 +5,6 @@ , openssl , gtk3 , stdenv -, rust }: rustPlatform.buildRustPackage rec { @@ -28,7 +27,7 @@ rustPlatform.buildRustPackage rec { # default installPhase don't install assets installPhase = '' runHook preInstall - make install PREFIX="$out" TARGET="target/${rust.toRustTarget stdenv.hostPlatform}/release/effitask" + make install PREFIX="$out" TARGET="target/${stdenv.hostPlatform.rust.rustcTarget}/release/effitask" runHook postInstall ''; diff --git a/pkgs/applications/window-managers/cosmic/applets/default.nix b/pkgs/applications/window-managers/cosmic/applets/default.nix index f64f6b31247..87da5b580e2 100644 --- a/pkgs/applications/window-managers/cosmic/applets/default.nix +++ b/pkgs/applications/window-managers/cosmic/applets/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, rust, rustPlatform +{ lib, stdenv, fetchFromGitHub, rustPlatform , cargo, just, pkg-config, util-linuxMinimal , dbus, glib, libxkbcommon, pulseaudio, wayland }: @@ -41,11 +41,11 @@ rustPlatform.buildRustPackage { justFlags = [ "--set" "prefix" (placeholder "out") - "--set" "target" "${rust.lib.toRustTargetSpecShort stdenv.hostPlatform}/release" + "--set" "target" "${stdenv.hostPlatform.rust.cargoShortTarget}/release" ]; # Force linking to libwayland-client, which is always dlopen()ed. - "CARGO_TARGET_${rust.toRustTargetForUseInEnvVars stdenv.hostPlatform}_RUSTFLAGS" = + "CARGO_TARGET_${stdenv.hostPlatform.rust.cargoEnvVarTarget}_RUSTFLAGS" = map (a: "-C link-arg=${a}") [ "-Wl,--push-state,--no-as-needed" "-lwayland-client" diff --git a/pkgs/applications/window-managers/cosmic/panel/default.nix b/pkgs/applications/window-managers/cosmic/panel/default.nix index 7908eeb3057..e5000fe184e 100644 --- a/pkgs/applications/window-managers/cosmic/panel/default.nix +++ b/pkgs/applications/window-managers/cosmic/panel/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, cargo, just, pkg-config, rust, rustPlatform +{ lib, stdenv, fetchFromGitHub, cargo, just, pkg-config, rustPlatform , libglvnd, libxkbcommon, wayland }: @@ -33,11 +33,11 @@ rustPlatform.buildRustPackage { justFlags = [ "--set" "prefix" (placeholder "out") - "--set" "bin-src" "target/${rust.lib.toRustTargetSpecShort stdenv.hostPlatform}/release/cosmic-panel" + "--set" "bin-src" "target/${stdenv.hostPlatform.rust.cargoShortTarget}/release/cosmic-panel" ]; # Force linking to libEGL, which is always dlopen()ed. - "CARGO_TARGET_${rust.toRustTargetForUseInEnvVars stdenv.hostPlatform}_RUSTFLAGS" = + "CARGO_TARGET_${stdenv.hostPlatform.rust.cargoEnvVarTarget}_RUSTFLAGS" = map (a: "-C link-arg=${a}") [ "-Wl,--push-state,--no-as-needed" "-lEGL" diff --git a/pkgs/applications/window-managers/cosmic/settings/default.nix b/pkgs/applications/window-managers/cosmic/settings/default.nix index 6767f963846..100d1ec316f 100644 --- a/pkgs/applications/window-managers/cosmic/settings/default.nix +++ b/pkgs/applications/window-managers/cosmic/settings/default.nix @@ -61,7 +61,7 @@ rustPlatform.buildRustPackage { (placeholder "out") "--set" "bin-src" - "target/${rust.lib.toRustTargetSpecShort stdenv.hostPlatform}/release/cosmic-settings" + "target/${stdenv.hostPlatform.rust.cargoShortTarget}/release/cosmic-settings" ]; meta = with lib; { diff --git a/pkgs/build-support/rust/build-rust-crate/build-crate.nix b/pkgs/build-support/rust/build-rust-crate/build-crate.nix index 944434472b0..e842b6a3f50 100644 --- a/pkgs/build-support/rust/build-rust-crate/build-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/build-crate.nix @@ -1,6 +1,5 @@ { lib, stdenv , mkRustcDepArgs, mkRustcFeatureArgs, needUnstableCLI -, rust }: { crateName, @@ -21,7 +20,7 @@ (mkRustcDepArgs dependencies crateRenames) (mkRustcFeatureArgs crateFeatures) ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--target" (rust.toRustTargetSpec stdenv.hostPlatform) + "--target" stdenv.hostPlatform.rust.rustcTargetSpec ] ++ lib.optionals (needUnstableCLI dependencies) [ "-Z" "unstable-options" ] ++ extraRustcOpts diff --git a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix index 60310f17874..a6f2528a82b 100644 --- a/pkgs/build-support/rust/build-rust-crate/configure-crate.nix +++ b/pkgs/build-support/rust/build-rust-crate/configure-crate.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: +{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs, mkRustcFeatureArgs }: { build , buildDependencies @@ -124,8 +124,8 @@ in '' export CARGO_PKG_AUTHORS="${authors}" export CARGO_PKG_DESCRIPTION="${crateDescription}" - export CARGO_CFG_TARGET_ARCH=${rust.toTargetArch stdenv.hostPlatform} - export CARGO_CFG_TARGET_OS=${rust.toTargetOs stdenv.hostPlatform} + export CARGO_CFG_TARGET_ARCH=${stdenv.hostPlatform.rust.platform.arch} + export CARGO_CFG_TARGET_OS=${stdenv.hostPlatform.rust.platform.os} export CARGO_CFG_TARGET_FAMILY="unix" export CARGO_CFG_UNIX=1 export CARGO_CFG_TARGET_ENV="gnu" @@ -136,8 +136,8 @@ in '' export CARGO_MANIFEST_DIR=$(pwd) export DEBUG="${toString (!release)}" export OPT_LEVEL="${toString optLevel}" - export TARGET="${rust.toRustTargetSpec stdenv.hostPlatform}" - export HOST="${rust.toRustTargetSpec stdenv.buildPlatform}" + export TARGET="${stdenv.hostPlatform.rust.rustcTargetSpec}" + export HOST="${stdenv.buildPlatform.rust.rustcTargetSpec}" export PROFILE=${if release then "release" else "debug"} export OUT_DIR=$(pwd)/target/build/${crateName}.out export CARGO_PKG_VERSION_MAJOR=${lib.elemAt version 0} diff --git a/pkgs/build-support/rust/build-rust-crate/default.nix b/pkgs/build-support/rust/build-rust-crate/default.nix index d977fb9f7c2..d569142af66 100644 --- a/pkgs/build-support/rust/build-rust-crate/default.nix +++ b/pkgs/build-support/rust/build-rust-crate/default.nix @@ -10,7 +10,6 @@ , fetchCrate , pkgsBuildBuild , rustc -, rust , cargo , jq , libiconv @@ -71,18 +70,14 @@ let inherit (import ./log.nix { inherit lib; }) noisily echo_colored; configureCrate = import ./configure-crate.nix { - inherit lib stdenv rust echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; + inherit lib stdenv echo_colored noisily mkRustcDepArgs mkRustcFeatureArgs; }; buildCrate = import ./build-crate.nix { - inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI rust; + inherit lib stdenv mkRustcDepArgs mkRustcFeatureArgs needUnstableCLI; }; installCrate = import ./install-crate.nix { inherit stdenv; }; - - # Allow access to the rust attribute set from inside buildRustCrate, which - # has a parameter that shadows the name. - rustAttrs = rust; in /* The overridable pkgs.buildRustCrate function. @@ -310,7 +305,7 @@ crate_: lib.makeOverridable depsMetadata = lib.foldl' (str: dep: str + dep.metadata) "" (dependencies ++ buildDependencies); hashedMetadata = builtins.hashString "sha256" (crateName + "-" + crateVersion + "___" + toString (mkRustcFeatureArgs crateFeatures) + - "___" + depsMetadata + "___" + rustAttrs.toRustTarget stdenv.hostPlatform); + "___" + depsMetadata + "___" + stdenv.hostPlatform.rust.rustcTarget); in lib.substring 0 10 hashedMetadata; diff --git a/pkgs/build-support/rust/build-rust-package/default.nix b/pkgs/build-support/rust/build-rust-package/default.nix index 0b28121a5b7..cf2ddbd084b 100644 --- a/pkgs/build-support/rust/build-rust-package/default.nix +++ b/pkgs/build-support/rust/build-rust-package/default.nix @@ -1,7 +1,6 @@ { lib , importCargoLock , fetchCargoTarball -, rust , stdenv , callPackage , cargoBuildHook @@ -78,13 +77,13 @@ let sha256 = args.cargoSha256; } // depsExtraArgs); - target = rust.toRustTargetSpec stdenv.hostPlatform; + target = stdenv.hostPlatform.rust.rustcTargetSpec; targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; sysroot = callPackage ./sysroot { } { inherit target; - shortTarget = rust.lib.toRustTargetSpecShort stdenv.hostPlatform; + shortTarget = stdenv.hostPlatform.rust.cargoShortTarget; RUSTFLAGS = args.RUSTFLAGS or ""; originalCargoToml = src + /Cargo.toml; # profile info is later extracted }; diff --git a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix index a6d53056d9c..bb95b7bdc35 100644 --- a/pkgs/build-support/rust/build-rust-package/sysroot/default.nix +++ b/pkgs/build-support/rust/build-rust-package/sysroot/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, rust, rustPlatform, buildPackages }: +{ lib, stdenv, rustPlatform, buildPackages }: { shortTarget, originalCargoToml, target, RUSTFLAGS }: @@ -26,7 +26,7 @@ in rustPlatform.buildRustPackage { done export RUST_SYSROOT=$(rustc --print=sysroot) - host=${rust.toRustTarget stdenv.buildPlatform} + host=${stdenv.buildPlatform.rust.rustcTarget} cp -r $RUST_SYSROOT/lib/rustlib/$host $out ''; diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix index 205d085d350..7703ff4abad 100644 --- a/pkgs/build-support/rust/hooks/default.nix +++ b/pkgs/build-support/rust/hooks/default.nix @@ -13,7 +13,7 @@ # This confusingly-named parameter indicates the *subdirectory of # `target/` from which to copy the build artifacts. It is derived # from a stdenv platform (or a JSON file). -, target ? rust.lib.toRustTargetSpecShort stdenv.hostPlatform +, target ? stdenv.hostPlatform.rust.cargoShortTarget }: { @@ -65,10 +65,10 @@ diff = "${lib.getBin buildPackages.diffutils}/bin/diff"; cargoConfig = '' - [target."${rust.toRustTarget stdenv.buildPlatform}"] + [target."${stdenv.buildPlatform.rust.rustcTarget}"] "linker" = "${rust.envVars.ccForBuild}" ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${rust.toRustTarget stdenv.hostPlatform}"] + [target."${stdenv.hostPlatform.rust.rustcTarget}"] "linker" = "${rust.envVars.ccForHost}" ''} "rustflags" = [ "-C", "target-feature=${if stdenv.hostPlatform.isStatic then "+" else "-"}crt-static" ] diff --git a/pkgs/build-support/rust/lib/default.nix b/pkgs/build-support/rust/lib/default.nix index ceca7323176..dad8ab52823 100644 --- a/pkgs/build-support/rust/lib/default.nix +++ b/pkgs/build-support/rust/lib/default.nix @@ -5,89 +5,6 @@ }: rec { - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_arch - toTargetArch = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.arch - else if platform.isAarch32 then "arm" - else if platform.isMips64 then "mips64" # never add "el" suffix - else if platform.isPower64 then "powerpc64" # never add "le" suffix - else platform.parsed.cpu.name; - - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_os - toTargetOs = platform: - /**/ if platform ? rustc.platform then platform.rustc.platform.os or "none" - else if platform.isDarwin then "macos" - else platform.parsed.kernel.name; - - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_family - toTargetFamily = platform: - if platform ? rustc.platform.target-family - then - ( - # Since https://github.com/rust-lang/rust/pull/84072 - # `target-family` is a list instead of single value. - let - f = platform.rustc.platform.target-family; - in - if builtins.isList f then f else [ f ] - ) - else lib.optional platform.isUnix "unix" - ++ lib.optional platform.isWindows "windows"; - - # https://doc.rust-lang.org/reference/conditional-compilation.html#target_vendor - toTargetVendor = platform: let - inherit (platform.parsed) vendor; - in platform.rustc.platform.vendor or { - "w64" = "pc"; - }.${vendor.name} or vendor.name; - - # Returns the name of the rust target, even if it is custom. Adjustments are - # because rust has slightly different naming conventions than we do. - toRustTarget = platform: let - inherit (platform.parsed) cpu kernel abi; - cpu_ = platform.rustc.platform.arch or { - "armv7a" = "armv7"; - "armv7l" = "armv7"; - "armv6l" = "arm"; - "armv5tel" = "armv5te"; - "riscv64" = "riscv64gc"; - }.${cpu.name} or cpu.name; - vendor_ = toTargetVendor platform; - in platform.rustc.config - or "${cpu_}-${vendor_}-${kernel.name}${lib.optionalString (abi.name != "unknown") "-${abi.name}"}"; - - # Returns the name of the rust target if it is standard, or the json file - # containing the custom target spec. - toRustTargetSpec = platform: - if platform ? rustc.platform - then builtins.toFile (toRustTarget platform + ".json") (builtins.toJSON platform.rustc.platform) - else toRustTarget platform; - - # Returns the name of the rust target if it is standard, or the - # basename of the file containing the custom target spec, without - # the .json extension. - # - # This is the name used by Cargo for target subdirectories. - toRustTargetSpecShort = platform: - lib.removeSuffix ".json" - (baseNameOf "${toRustTargetSpec platform}"); - - # When used as part of an environment variable name, triples are - # uppercased and have all hyphens replaced by underscores: - # - # https://github.com/rust-lang/cargo/pull/9169 - # https://github.com/rust-lang/cargo/issues/8285#issuecomment-634202431 - # - toRustTargetForUseInEnvVars = platform: - lib.strings.replaceStrings ["-"] ["_"] - (lib.strings.toUpper - (toRustTargetSpecShort platform)); - - # Returns true if the target is no_std - # https://github.com/rust-lang/rust/blob/2e44c17c12cec45b6a682b1e53a04ac5b5fcc9d2/src/bootstrap/config.rs#L415-L421 - IsNoStdTarget = platform: let rustTarget = toRustTarget platform; in - builtins.any (t: lib.hasInfix t rustTarget) ["-none" "nvptx" "switch" "-uefi"]; - # These environment variables must be set when using `cargo-c` and # several other tools which do not deal well with cross # compilation. The symptom of the problem they fix is errors due @@ -107,12 +24,12 @@ rec { ccForTarget = "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc"; cxxForTarget = "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++"; - rustBuildPlatform = toRustTarget stdenv.buildPlatform; - rustBuildPlatformSpec = toRustTargetSpec stdenv.buildPlatform; - rustHostPlatform = toRustTarget stdenv.hostPlatform; - rustHostPlatformSpec = toRustTargetSpec stdenv.hostPlatform; - rustTargetPlatform = toRustTarget stdenv.targetPlatform; - rustTargetPlatformSpec = toRustTargetSpec stdenv.targetPlatform; + rustBuildPlatform = stdenv.buildPlatform.rust.rustcTarget; + rustBuildPlatformSpec = stdenv.buildPlatform.rust.rustcTargetSpec; + rustHostPlatform = stdenv.hostPlatform.rust.rustcTarget; + rustHostPlatformSpec = stdenv.hostPlatform.rust.rustcTargetSpec; + rustTargetPlatform = stdenv.targetPlatform.rust.rustcTarget; + rustTargetPlatformSpec = stdenv.targetPlatform.rust.rustcTargetSpec; in { inherit ccForBuild cxxForBuild rustBuildPlatform rustBuildPlatformSpec @@ -131,20 +48,34 @@ rec { # the following lines when rustTargetPlatform collides with # rustHostPlatform. + lib.optionalString (rustTargetPlatform != rustHostPlatform) '' - "CC_${toRustTargetForUseInEnvVars stdenv.targetPlatform}=${ccForTarget}" \ - "CXX_${toRustTargetForUseInEnvVars stdenv.targetPlatform}=${cxxForTarget}" \ - "CARGO_TARGET_${toRustTargetForUseInEnvVars stdenv.targetPlatform}_LINKER=${ccForTarget}" \ + "CC_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${ccForTarget}" \ + "CXX_${stdenv.targetPlatform.rust.cargoEnvVarTarget}=${cxxForTarget}" \ + "CARGO_TARGET_${stdenv.targetPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForTarget}" \ '' + '' - "CC_${toRustTargetForUseInEnvVars stdenv.hostPlatform}=${ccForHost}" \ - "CXX_${toRustTargetForUseInEnvVars stdenv.hostPlatform}=${cxxForHost}" \ - "CARGO_TARGET_${toRustTargetForUseInEnvVars stdenv.hostPlatform}_LINKER=${ccForHost}" \ + "CC_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${ccForHost}" \ + "CXX_${stdenv.hostPlatform.rust.cargoEnvVarTarget}=${cxxForHost}" \ + "CARGO_TARGET_${stdenv.hostPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForHost}" \ '' + '' - "CC_${toRustTargetForUseInEnvVars stdenv.buildPlatform}=${ccForBuild}" \ - "CXX_${toRustTargetForUseInEnvVars stdenv.buildPlatform}=${cxxForBuild}" \ - "CARGO_TARGET_${toRustTargetForUseInEnvVars stdenv.buildPlatform}_LINKER=${ccForBuild}" \ + "CC_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${ccForBuild}" \ + "CXX_${stdenv.buildPlatform.rust.cargoEnvVarTarget}=${cxxForBuild}" \ + "CARGO_TARGET_${stdenv.buildPlatform.rust.cargoEnvVarTarget}_LINKER=${ccForBuild}" \ "CARGO_BUILD_TARGET=${rustBuildPlatform}" \ "HOST_CC=${buildPackages.stdenv.cc}/bin/cc" \ "HOST_CXX=${buildPackages.stdenv.cc}/bin/c++" \ ''; }; +} // lib.mapAttrs (old: new: platform: + # TODO: enable warning after 23.05 is EOL. + # lib.warn "`rust.${old} platform` is deprecated. Use `platform.rust.${new}` instead." + lib.getAttrFromPath new platform.rust) +{ + toTargetArch = [ "platform" "arch" ]; + toTargetOs = [ "platform" "os" ]; + toTargetFamily = [ "platform" "target-family" ]; + toTargetVendor = [ "platform" "vendor" ]; + toRustTarget = [ "rustcTarget" ]; + toRustTargetSpec = [ "rustcTargetSpec" ]; + toRustTargetSpecShort = [ "cargoShortTarget" ]; + toRustTargetForUseInEnvVars = [ "cargoEnvVarTarget" ]; + IsNoStdTarget = [ "isNoStdTarget" ]; } diff --git a/pkgs/desktops/gnome/core/gnome-tour/default.nix b/pkgs/desktops/gnome/core/gnome-tour/default.nix index c471417d616..26ac73f546d 100644 --- a/pkgs/desktops/gnome/core/gnome-tour/default.nix +++ b/pkgs/desktops/gnome/core/gnome-tour/default.nix @@ -17,7 +17,6 @@ , libadwaita , librsvg , rustc -, rust , writeText , cargo }: @@ -65,7 +64,7 @@ stdenv.mkDerivation rec { # ERROR: 'rust' compiler binary not defined in cross or native file crossFile = writeText "cross-file.conf" '' [binaries] - rust = [ 'rustc', '--target', '${rust.toRustTargetSpec stdenv.hostPlatform}' ] + rust = [ 'rustc', '--target', '${stdenv.hostPlatform.rust.rustcTargetSpec}' ] ''; in lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ "--cross-file=${crossFile}" ]; diff --git a/pkgs/development/compilers/mrustc/bootstrap.nix b/pkgs/development/compilers/mrustc/bootstrap.nix index a8bf26bacdf..0996702ec62 100644 --- a/pkgs/development/compilers/mrustc/bootstrap.nix +++ b/pkgs/development/compilers/mrustc/bootstrap.nix @@ -2,7 +2,6 @@ , fetchurl , mrustc , mrustc-minicargo -, rust , llvm_12 , llvmPackages_12 , libffi @@ -74,7 +73,7 @@ stdenv.mkDerivation rec { "MRUSTC=${mrustc}/bin/mrustc" #"MINICARGO=${mrustc-minicargo}/bin/minicargo" # FIXME: we need to rebuild minicargo locally so --manifest-overrides is applied "LLVM_CONFIG=${llvm_12.dev}/bin/llvm-config" - "RUSTC_TARGET=${rust.toRustTarget stdenv.targetPlatform}" + "RUSTC_TARGET=${stdenv.targetPlatform.rust.rustcTarget}" ]; buildPhase = '' @@ -129,7 +128,7 @@ stdenv.mkDerivation rec { cp run_rustc/${outputDir}/prefix/bin/rustc_binary $out/bin/rustc cp -r run_rustc/${outputDir}/prefix/lib/* $out/lib/ - cp $out/lib/rustlib/${rust.toRustTarget stdenv.targetPlatform}/lib/*.so $out/lib/ + cp $out/lib/rustlib/${stdenv.targetPlatform.rust.rustcTarget}/lib/*.so $out/lib/ runHook postInstall ''; @@ -146,4 +145,3 @@ stdenv.mkDerivation rec { platforms = [ "x86_64-linux" ]; }; } - diff --git a/pkgs/development/compilers/rust/bootstrap.nix b/pkgs/development/compilers/rust/bootstrap.nix index 7c0b46d5593..97ac6927025 100644 --- a/pkgs/development/compilers/rust/bootstrap.nix +++ b/pkgs/development/compilers/rust/bootstrap.nix @@ -1,7 +1,7 @@ -{ stdenv, fetchurl, rust, callPackage, version, hashes }: +{ stdenv, fetchurl, callPackage, version, hashes }: let - platform = rust.toRustTarget stdenv.hostPlatform; + platform = stdenv.hostPlatform.rust.rustcTarget; src = fetchurl { url = "https://static.rust-lang.org/dist/rust-${version}-${platform}.tar.gz"; diff --git a/pkgs/development/compilers/rust/cargo.nix b/pkgs/development/compilers/rust/cargo.nix index 46e238ff479..8eb628aa536 100644 --- a/pkgs/development/compilers/rust/cargo.nix +++ b/pkgs/development/compilers/rust/cargo.nix @@ -1,6 +1,6 @@ { lib, stdenv, pkgsBuildHost, pkgsHostHost , file, curl, pkg-config, python3, openssl, cmake, zlib -, installShellFiles, makeWrapper, rustPlatform, rust, rustc +, installShellFiles, makeWrapper, rustPlatform, rustc , CoreFoundation, Security , auditable ? !cargo-auditable.meta.broken , cargo-auditable @@ -116,6 +116,6 @@ rustPlatform.buildRustPackage.override { broken = stdenv.hostPlatform.isx86 && stdenv.buildPlatform != stdenv.hostPlatform; }; } -// lib.optionalAttrs (rust.toRustTarget stdenv.buildPlatform != rust.toRustTarget stdenv.hostPlatform) { +// lib.optionalAttrs (stdenv.buildPlatform.rust.rustcTarget != stdenv.hostPlatform.rust.rustcTarget) { HOST_PKG_CONFIG_PATH="${pkgsBuildBuild.pkg-config}/bin/pkg-config"; }) diff --git a/pkgs/development/compilers/rust/rustc.nix b/pkgs/development/compilers/rust/rustc.nix index 44c290acbd3..5f97b0f4019 100644 --- a/pkgs/development/compilers/rust/rustc.nix +++ b/pkgs/development/compilers/rust/rustc.nix @@ -1,7 +1,7 @@ { lib, stdenv, removeReferencesTo, pkgsBuildBuild, pkgsBuildHost, pkgsBuildTarget, targetPackages , llvmShared, llvmSharedForBuild, llvmSharedForHost, llvmSharedForTarget, llvmPackages , fetchurl, file, python3 -, darwin, cargo, cmake, rust, rustc +, darwin, cargo, cmake, rustc , pkg-config, openssl, xz , libiconv , which, libffi @@ -51,7 +51,7 @@ in stdenv.mkDerivation rec { # but it does support checking these idiosyncratic PKG_CONFIG_${TRIPLE} # environment variables. # [1]: https://github.com/rust-lang/pkg-config-rs/issues/53 - "PKG_CONFIG_${builtins.replaceStrings ["-"] ["_"] (rust.toRustTarget stdenv.buildPlatform)}" = + "PKG_CONFIG_${builtins.replaceStrings ["-"] ["_"] stdenv.buildPlatform.rust.rustcTarget}" = "${pkgsBuildHost.stdenv.cc.targetPrefix}pkg-config"; NIX_LDFLAGS = toString ( @@ -69,9 +69,9 @@ in stdenv.mkDerivation rec { prefixForStdenv = stdenv: "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}"; ccPrefixForStdenv = stdenv: "${prefixForStdenv stdenv}${if (stdenv.cc.isClang or false) then "clang" else "cc"}"; cxxPrefixForStdenv = stdenv: "${prefixForStdenv stdenv}${if (stdenv.cc.isClang or false) then "clang++" else "c++"}"; - setBuild = "--set=target.${rust.toRustTarget stdenv.buildPlatform}"; - setHost = "--set=target.${rust.toRustTarget stdenv.hostPlatform}"; - setTarget = "--set=target.${rust.toRustTarget stdenv.targetPlatform}"; + setBuild = "--set=target.${stdenv.buildPlatform.rust.rustcTarget}"; + setHost = "--set=target.${stdenv.hostPlatform.rust.rustcTarget}"; + setTarget = "--set=target.${stdenv.targetPlatform.rust.rustcTarget}"; ccForBuild = ccPrefixForStdenv pkgsBuildBuild.targetPackages.stdenv; cxxForBuild = cxxPrefixForStdenv pkgsBuildBuild.targetPackages.stdenv; ccForHost = ccPrefixForStdenv pkgsBuildHost.targetPackages.stdenv; @@ -85,23 +85,23 @@ in stdenv.mkDerivation rec { "--tools=rustc,rust-analyzer-proc-macro-srv" "--enable-rpath" "--enable-vendor" - "--build=${rust.toRustTargetSpec stdenv.buildPlatform}" - "--host=${rust.toRustTargetSpec stdenv.hostPlatform}" + "--build=${stdenv.buildPlatform.rust.rustcTargetSpec}" + "--host=${stdenv.hostPlatform.rust.rustcTargetSpec}" # std is built for all platforms in --target. "--target=${concatStringsSep "," ([ - (rust.toRustTargetSpec stdenv.targetPlatform) + stdenv.targetPlatform.rust.rustcTargetSpec # (build!=target): When cross-building a compiler we need to add # the build platform as well so rustc can compile build.rs # scripts. ] ++ optionals (stdenv.buildPlatform != stdenv.targetPlatform && !fastCross) [ - (rust.toRustTargetSpec stdenv.buildPlatform) + stdenv.buildPlatform.rust.rustcTargetSpec # (host!=target): When building a cross-targeting compiler we # need to add the host platform as well so rustc can compile # build.rs scripts. ] ++ optionals (stdenv.hostPlatform != stdenv.targetPlatform && !fastCross) [ - (rust.toRustTargetSpec stdenv.hostPlatform) + stdenv.hostPlatform.rust.rustcTargetSpec ])}" "${setBuild}.cc=${ccForBuild}" @@ -132,7 +132,7 @@ in stdenv.mkDerivation rec { "${setHost}.musl-root=${pkgsBuildHost.targetPackages.stdenv.cc.libc}" ] ++ optionals stdenv.targetPlatform.isMusl [ "${setTarget}.musl-root=${pkgsBuildTarget.targetPackages.stdenv.cc.libc}" - ] ++ optionals (rust.IsNoStdTarget stdenv.targetPlatform) [ + ] ++ optionals stdenv.targetPlatform.rust.isNoStdTarget [ "--disable-docs" ] ++ optionals (stdenv.isDarwin && stdenv.isx86_64) [ # https://github.com/rust-lang/rust/issues/92173 @@ -144,12 +144,12 @@ in stdenv.mkDerivation rec { buildPhase = if fastCross then " runHook preBuild - mkdir -p build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-{std,rustc}/${rust.toRustTargetSpec stdenv.hostPlatform}/release/ - ln -s ${rustc}/lib/rustlib/${rust.toRustTargetSpec stdenv.hostPlatform}/libstd-*.so build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-std/${rust.toRustTargetSpec stdenv.hostPlatform}/release/libstd.so - ln -s ${rustc}/lib/rustlib/${rust.toRustTargetSpec stdenv.hostPlatform}/librustc_driver-*.so build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-rustc/${rust.toRustTargetSpec stdenv.hostPlatform}/release/librustc.so - ln -s ${rustc}/bin/rustc build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-rustc/${rust.toRustTargetSpec stdenv.hostPlatform}/release/rustc-main - touch build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-std/${rust.toRustTargetSpec stdenv.hostPlatform}/release/.libstd.stamp - touch build/${rust.toRustTargetSpec stdenv.hostPlatform}/stage0-rustc/${rust.toRustTargetSpec stdenv.hostPlatform}/release/.librustc.stamp + mkdir -p build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-{std,rustc}/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/ + ln -s ${rustc}/lib/rustlib/${stdenv.hostPlatform.rust.rustcTargetSpec}/libstd-*.so build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-std/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/libstd.so + ln -s ${rustc}/lib/rustlib/${stdenv.hostPlatform.rust.rustcTargetSpec}/librustc_driver-*.so build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-rustc/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/librustc.so + ln -s ${rustc}/bin/rustc build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-rustc/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/rustc-main + touch build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-std/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/.libstd.stamp + touch build/${stdenv.hostPlatform.rust.rustcTargetSpec}/stage0-rustc/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/.librustc.stamp python ./x.py --keep-stage=0 --stage=1 build library/std runHook postBuild @@ -162,8 +162,8 @@ in stdenv.mkDerivation rec { mkdir -v $out/bin $doc $man makeWrapper ${rustc}/bin/rustc $out/bin/rustc --add-flags "--sysroot $out" makeWrapper ${rustc}/bin/rustdoc $out/bin/rustdoc --add-flags "--sysroot $out" - ln -s ${rustc}/lib/rustlib/{manifest-rust-std-,}${rust.toRustTargetSpec stdenv.hostPlatform} $out/lib/rustlib/ - echo rust-std-${rust.toRustTargetSpec stdenv.hostPlatform} >> $out/lib/rustlib/components + ln -s ${rustc}/lib/rustlib/{manifest-rust-std-,}${stdenv.hostPlatform.rust.rustcTargetSpec} $out/lib/rustlib/ + echo rust-std-${stdenv.hostPlatform.rust.rustcTargetSpec} >> $out/lib/rustlib/components lndir ${rustc.doc} $doc lndir ${rustc.man} $man diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index a907cda5f11..f7cc10274ae 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -183,16 +183,14 @@ in { }; } ./setuptools-check-hook.sh) {}; - setuptoolsRustBuildHook = callPackage ({ makePythonHook, setuptools-rust, rust }: + setuptoolsRustBuildHook = callPackage ({ makePythonHook, setuptools-rust }: makePythonHook { name = "setuptools-rust-setup-hook"; propagatedBuildInputs = [ setuptools-rust ]; substitutions = { pyLibDir = "${python}/lib/${python.libPrefix}"; - cargoBuildTarget = rust.toRustTargetSpec stdenv.hostPlatform; - cargoLinkerVar = lib.toUpper ( - builtins.replaceStrings ["-"] ["_"] ( - rust.toRustTarget stdenv.hostPlatform)); + cargoBuildTarget = stdenv.hostPlatform.rust.rustcTargetSpec; + cargoLinkerVar = stdenv.hostPlatform.rust.cargoEnvVarTarget; targetLinker = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; }; } ./setuptools-rust-hook.sh) {}; diff --git a/pkgs/development/libraries/gstreamer/rs/default.nix b/pkgs/development/libraries/gstreamer/rs/default.nix index e0d324936dd..1d06f880959 100644 --- a/pkgs/development/libraries/gstreamer/rs/default.nix +++ b/pkgs/development/libraries/gstreamer/rs/default.nix @@ -8,7 +8,6 @@ , ninja , python3 , pkg-config -, rust , rustc , cargo , cargo-c @@ -208,7 +207,7 @@ stdenv.mkDerivation rec { ] ++ (let crossFile = writeText "cross-file.conf" '' [binaries] - rust = [ 'rustc', '--target', '${rust.toRustTargetSpec stdenv.hostPlatform}' ] + rust = [ 'rustc', '--target', '${stdenv.hostPlatform.rust.rustcTargetSpec}' ] ''; in lib.optionals (stdenv.buildPlatform != stdenv.hostPlatform) [ "--cross-file=${crossFile}" diff --git a/pkgs/development/libraries/libdovi/default.nix b/pkgs/development/libraries/libdovi/default.nix index cf1a9a561f3..c70a217f6fc 100644 --- a/pkgs/development/libraries/libdovi/default.nix +++ b/pkgs/development/libraries/libdovi/default.nix @@ -5,9 +5,7 @@ , rust , stdenv }: -let - rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; -in + rustPlatform.buildRustPackage rec { pname = "libdovi"; version = "3.1.2"; @@ -28,19 +26,19 @@ rustPlatform.buildRustPackage rec { buildPhase = '' runHook preBuild - ${rust.envVars.setEnv} cargo cbuild -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cbuild -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} runHook postBuild ''; installPhase = '' runHook preInstall - ${rust.envVars.setEnv} cargo cinstall -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cinstall -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} runHook postInstall ''; checkPhase = '' runHook preCheck - ${rust.envVars.setEnv} cargo ctest -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo ctest -j $NIX_BUILD_CORES --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} runHook postCheck ''; diff --git a/pkgs/development/libraries/libimagequant/default.nix b/pkgs/development/libraries/libimagequant/default.nix index 0ac6e89e23e..f60a1c7cab7 100644 --- a/pkgs/development/libraries/libimagequant/default.nix +++ b/pkgs/development/libraries/libimagequant/default.nix @@ -1,8 +1,5 @@ { lib, stdenv, fetchFromGitHub, fetchurl, rust, rustPlatform, cargo-c, python3 }: -let - rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; -in rustPlatform.buildRustPackage rec { pname = "libimagequant"; version = "4.2.2"; @@ -26,13 +23,13 @@ rustPlatform.buildRustPackage rec { postBuild = '' pushd imagequant-sys - ${rust.envVars.setEnv} cargo cbuild --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cbuild --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} popd ''; postInstall = '' pushd imagequant-sys - ${rust.envVars.setEnv} cargo cinstall --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cinstall --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} popd ''; diff --git a/pkgs/development/libraries/librsvg/default.nix b/pkgs/development/libraries/librsvg/default.nix index a0843d4b2ef..7ba6f1963e7 100644 --- a/pkgs/development/libraries/librsvg/default.nix +++ b/pkgs/development/libraries/librsvg/default.nix @@ -15,7 +15,6 @@ , libobjc , rustPlatform , rustc -, rust , cargo-auditable-cargo-wrapper , gi-docgen , python3Packages @@ -106,7 +105,7 @@ stdenv.mkDerivation (finalAttrs: { "--enable-always-build-tests" ] ++ lib.optional stdenv.isDarwin "--disable-Bsymbolic" - ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) "RUST_TARGET=${rust.toRustTarget stdenv.hostPlatform}"; + ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) "RUST_TARGET=${stdenv.hostPlatform.rust.rustcTarget}"; doCheck = false; # all tests fail on libtool-generated rsvg-convert not being able to find coreutils diff --git a/pkgs/development/libraries/relibc/default.nix b/pkgs/development/libraries/relibc/default.nix index 28ed971a25e..3bc476ccc0f 100644 --- a/pkgs/development/libraries/relibc/default.nix +++ b/pkgs/development/libraries/relibc/default.nix @@ -64,7 +64,7 @@ redoxRustPlatform.buildRustPackage rec { ''; # TODO: should be hostPlatform - TARGET = buildPackages.rust.toRustTargetSpec stdenvNoCC.targetPlatform; + TARGET = stdenvNoCC.targetPlatform.rust.rustcTargetSpec; cargoLock = { lockFile = ./Cargo.lock; diff --git a/pkgs/development/libraries/rustc-demangle/default.nix b/pkgs/development/libraries/rustc-demangle/default.nix index 6dd6d46e30b..4e142c58452 100644 --- a/pkgs/development/libraries/rustc-demangle/default.nix +++ b/pkgs/development/libraries/rustc-demangle/default.nix @@ -1,4 +1,4 @@ -{ rustPlatform, fetchFromGitHub, rust, lib, stdenv }: +{ rustPlatform, fetchFromGitHub, lib, stdenv }: rustPlatform.buildRustPackage rec { pname = "rustc-demangle"; @@ -24,7 +24,7 @@ rustPlatform.buildRustPackage rec { postInstall = '' mkdir -p $out/lib - cp target/${rust.toRustTargetSpec stdenv.hostPlatform}/release/librustc_demangle.so $out/lib + cp target/${stdenv.hostPlatform.rust.rustcTargetSpec}/release/librustc_demangle.so $out/lib cp -R crates/capi/include $out ''; diff --git a/pkgs/development/tools/lalrpop/default.nix b/pkgs/development/tools/lalrpop/default.nix index 0bc8bddefbd..32e6ec7a35b 100644 --- a/pkgs/development/tools/lalrpop/default.nix +++ b/pkgs/development/tools/lalrpop/default.nix @@ -1,6 +1,5 @@ { lib , rustPlatform -, rust , fetchFromGitHub , substituteAll , stdenv @@ -24,7 +23,7 @@ rustPlatform.buildRustPackage rec { patches = [ (substituteAll { src = ./use-correct-binary-path-in-tests.patch; - target_triple = rust.toRustTarget stdenv.hostPlatform; + target_triple = stdenv.hostPlatform.rust.rustcTarget; }) ]; diff --git a/pkgs/development/tools/rust/cargo-benchcmp/default.nix b/pkgs/development/tools/rust/cargo-benchcmp/default.nix index 4aa3c411f12..d46a57355d7 100644 --- a/pkgs/development/tools/rust/cargo-benchcmp/default.nix +++ b/pkgs/development/tools/rust/cargo-benchcmp/default.nix @@ -2,7 +2,6 @@ , rustPlatform , fetchFromGitHub , substituteAll -, rust , stdenv }: @@ -23,7 +22,7 @@ rustPlatform.buildRustPackage rec { # patch the binary path so tests can find the binary when `--target` is present (substituteAll { src = ./fix-test-binary-path.patch; - shortTarget = rust.toRustTarget stdenv.hostPlatform; + shortTarget = stdenv.hostPlatform.rust.rustcTarget; }) ]; diff --git a/pkgs/development/tools/rust/cargo-watch/default.nix b/pkgs/development/tools/rust/cargo-watch/default.nix index fda47b5070a..63217b655ac 100644 --- a/pkgs/development/tools/rust/cargo-watch/default.nix +++ b/pkgs/development/tools/rust/cargo-watch/default.nix @@ -5,7 +5,6 @@ , Cocoa , CoreServices , Foundation -, rust , libiconv }: @@ -27,7 +26,7 @@ rustPlatform.buildRustPackage rec { # `test with_cargo` tries to call cargo-watch as a cargo subcommand # (calling cargo-watch with command `cargo watch`) preCheck = '' - export PATH="$(pwd)/target/${rust.toRustTarget stdenv.hostPlatform}/release:$PATH" + export PATH="$(pwd)/target/${stdenv.hostPlatform.rust.rustcTarget}/release:$PATH" ''; meta = with lib; { diff --git a/pkgs/development/web/deno/librusty_v8.nix b/pkgs/development/web/deno/librusty_v8.nix index 20fd20a5b98..e4d2a2fec70 100644 --- a/pkgs/development/web/deno/librusty_v8.nix +++ b/pkgs/development/web/deno/librusty_v8.nix @@ -1,11 +1,10 @@ # auto-generated file -- DO NOT EDIT! -{ rust, stdenv, fetchurl }: +{ stdenv, fetchurl }: let - arch = rust.toRustTarget stdenv.hostPlatform; fetch_librusty_v8 = args: fetchurl { name = "librusty_v8-${args.version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${arch}.a"; + url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${stdenv.hostPlatform.rust.rustcTarget}.a"; sha256 = args.shas.${stdenv.hostPlatform.system}; meta = { inherit (args) version; }; }; diff --git a/pkgs/development/web/deno/update/librusty_v8.ts b/pkgs/development/web/deno/update/librusty_v8.ts index b38e0a28f1a..39831f6ea27 100644 --- a/pkgs/development/web/deno/update/librusty_v8.ts +++ b/pkgs/development/web/deno/update/librusty_v8.ts @@ -40,13 +40,12 @@ fetchurl { const templateDeps = (version: string, deps: PrefetchResult[]) => `# auto-generated file -- DO NOT EDIT! -{ rust, stdenv, fetchurl }: +{ stdenv, fetchurl }: let - arch = rust.toRustTarget stdenv.hostPlatform; fetch_librusty_v8 = args: fetchurl { name = "librusty_v8-\${args.version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v\${args.version}/librusty_v8_release_\${arch}.a"; + url = "https://github.com/denoland/rusty_v8/releases/download/v\${args.version}/librusty_v8_release_\${stdenv.hostPlatform.rust.rustcTarget}.a"; sha256 = args.shas.\${stdenv.hostPlatform.system}; meta = { inherit (args) version; }; }; diff --git a/pkgs/development/web/edge-runtime/librusty_v8.nix b/pkgs/development/web/edge-runtime/librusty_v8.nix index 6e9fc0a669f..1e0a306c13b 100644 --- a/pkgs/development/web/edge-runtime/librusty_v8.nix +++ b/pkgs/development/web/edge-runtime/librusty_v8.nix @@ -1,11 +1,10 @@ # auto-generated file -- DO NOT EDIT! -{ rust, stdenv, fetchurl }: +{ stdenv, fetchurl }: let - arch = rust.toRustTarget stdenv.hostPlatform; fetch_librusty_v8 = args: fetchurl { name = "librusty_v8-${args.version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${arch}.a"; + url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${stdenv.hostPlatform.rust.rustcTarget}.a"; sha256 = args.shas.${stdenv.hostPlatform.system}; meta = { inherit (args) version; }; }; diff --git a/pkgs/servers/http/router/librusty_v8.nix b/pkgs/servers/http/router/librusty_v8.nix index 60c6b4caef8..3e3bf55c757 100644 --- a/pkgs/servers/http/router/librusty_v8.nix +++ b/pkgs/servers/http/router/librusty_v8.nix @@ -1,10 +1,9 @@ -{ rust, stdenv, fetchurl }: +{ stdenv, fetchurl }: let - arch = rust.toRustTarget stdenv.hostPlatform; fetch_librusty_v8 = args: fetchurl { name = "librusty_v8-${args.version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${arch}.a"; + url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${stdenv.hostPlatform.rust.rustcTarget}.a"; sha256 = args.shas.${stdenv.hostPlatform.system}; meta = { inherit (args) version; }; }; diff --git a/pkgs/servers/matrix-synapse/matrix-hookshot/default.nix b/pkgs/servers/matrix-synapse/matrix-hookshot/default.nix index 99b28a4defa..6113bd6ff41 100644 --- a/pkgs/servers/matrix-synapse/matrix-hookshot/default.nix +++ b/pkgs/servers/matrix-synapse/matrix-hookshot/default.nix @@ -5,7 +5,6 @@ , makeWrapper , matrix-sdk-crypto-nodejs , mkYarnPackage -, rust , cargo , rustPlatform , rustc @@ -60,7 +59,7 @@ mkYarnPackage rec { buildPhase = '' runHook preBuild cd deps/${pname} - napi build --target ${rust.toRustTargetSpec stdenv.targetPlatform} --dts ../src/libRs.d.ts --release ./lib + napi build --target ${stdenv.targetPlatform.rust.rustcTargetSpec} --dts ../src/libRs.d.ts --release ./lib yarn run build:app:fix-defs yarn run build:app yarn run build:web diff --git a/pkgs/servers/windmill/default.nix b/pkgs/servers/windmill/default.nix index 6b822ade08b..a10610ea195 100644 --- a/pkgs/servers/windmill/default.nix +++ b/pkgs/servers/windmill/default.nix @@ -16,7 +16,6 @@ , pixman , pkg-config , python3 -, rust , rustfmt , stdenv , swagger-cli @@ -70,11 +69,10 @@ rustPlatform.buildRustPackage { SQLX_OFFLINE = "true"; RUSTY_V8_ARCHIVE = let - arch = rust.toRustTarget stdenv.hostPlatform; fetch_librusty_v8 = args: fetchurl { name = "librusty_v8-${args.version}"; - url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${arch}.a"; + url = "https://github.com/denoland/rusty_v8/releases/download/v${args.version}/librusty_v8_release_${stdenv.hostPlatform.rust.rustcTarget}.a"; sha256 = args.shas.${stdenv.hostPlatform.system} or (throw "Unsupported platform ${stdenv.hostPlatform.system}"); meta = { inherit (args) version; }; }; diff --git a/pkgs/tools/misc/halp/default.nix b/pkgs/tools/misc/halp/default.nix index d9028a8392a..89867fe7a94 100644 --- a/pkgs/tools/misc/halp/default.nix +++ b/pkgs/tools/misc/halp/default.nix @@ -5,7 +5,6 @@ , stdenv , darwin , unixtools -, rust }: rustPlatform.buildRustPackage rec { @@ -48,7 +47,7 @@ rustPlatform.buildRustPackage rec { postPatch = '' substituteInPlace src/helper/args/mod.rs \ - --subst-var-by releaseDir target/${rust.toRustTargetSpec stdenv.hostPlatform}/$cargoCheckType + --subst-var-by releaseDir target/${stdenv.hostPlatform.rust.rustcTargetSpec}/$cargoCheckType ''; preCheck = '' diff --git a/pkgs/tools/system/zram-generator/default.nix b/pkgs/tools/system/zram-generator/default.nix index 81bf4082d5c..3d55d7c1961 100644 --- a/pkgs/tools/system/zram-generator/default.nix +++ b/pkgs/tools/system/zram-generator/default.nix @@ -1,7 +1,6 @@ { lib , stdenv , fetchFromGitHub -, rust , rustPlatform , pkg-config , ronn @@ -28,7 +27,7 @@ rustPlatform.buildRustPackage rec { postPatch = '' cp ${./Cargo.lock} Cargo.lock substituteInPlace Makefile \ - --replace 'target/$(BUILDTYPE)' 'target/${rust.toRustTargetSpec stdenv.hostPlatform}/$(BUILDTYPE)' + --replace 'target/$(BUILDTYPE)' 'target/${stdenv.hostPlatform.rust.rustcTargetSpec}/$(BUILDTYPE)' substituteInPlace src/generator.rs \ --replace 'Command::new("systemd-detect-virt")' 'Command::new("${systemd}/bin/systemd-detect-virt")' \ --replace 'Command::new("modprobe")' 'Command::new("${kmod}/bin/modprobe")' diff --git a/pkgs/tools/video/rav1e/default.nix b/pkgs/tools/video/rav1e/default.nix index 70e95cb4f01..146ff607254 100644 --- a/pkgs/tools/video/rav1e/default.nix +++ b/pkgs/tools/video/rav1e/default.nix @@ -13,10 +13,7 @@ , buildPackages }: -let - rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; - -in rustPlatform.buildRustPackage rec { +rustPlatform.buildRustPackage rec { pname = "rav1e"; version = "0.6.6"; @@ -47,11 +44,11 @@ in rustPlatform.buildRustPackage rec { checkType = "debug"; postBuild = '' - ${rust.envVars.setEnv} cargo cbuild --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cbuild --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} ''; postInstall = '' - ${rust.envVars.setEnv} cargo cinstall --release --frozen --prefix=${placeholder "out"} --target ${rustTargetPlatformSpec} + ${rust.envVars.setEnv} cargo cinstall --release --frozen --prefix=${placeholder "out"} --target ${stdenv.hostPlatform.rust.rustcTarget} ''; meta = with lib; { -- cgit 1.4.1 From 2099ebdf593da8babffb96f3909fb8568ce483fb Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Wed, 18 Oct 2023 16:50:59 +0200 Subject: lib: add asserts.assertEachOneOf Along the lines of `assertOneOf`, but expects a list of values to be compared. This gives a good error message and is useful for lists of values, like `supportedGhcVersions` in the arguments of `haskell-language-server`. --- lib/asserts.nix | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'lib') diff --git a/lib/asserts.nix b/lib/asserts.nix index 98e0b490acf..8d0a621f4c1 100644 --- a/lib/asserts.nix +++ b/lib/asserts.nix @@ -50,4 +50,33 @@ rec { lib.generators.toPretty {} xs}, but is: ${ lib.generators.toPretty {} val}"; + /* Specialized `assertMsg` for checking if every one of `vals` is one of the elements + of the list `xs`. Useful for checking lists of supported attributes. + + Example: + let sslLibraries = [ "libressl" "bearssl" ]; + in assertEachOneOf "sslLibraries" sslLibraries [ "openssl" "bearssl" ] + stderr> error: each element in sslLibraries must be one of [ + stderr> "openssl" + stderr> "bearssl" + stderr> ], but is: [ + stderr> "libressl" + stderr> "bearssl" + stderr> ] + + Type: + assertEachOneOf :: String -> List ComparableVal -> List ComparableVal -> Bool + */ + assertEachOneOf = + # The name of the variable the user entered `val` into, for inclusion in the error message + name: + # The list of values of what the user provided, to be compared against the values in `xs` + vals: + # The list of valid values + xs: + assertMsg + (lib.all (val: lib.elem val xs) vals) + "each element in ${name} must be one of ${ + lib.generators.toPretty {} xs}, but is: ${ + lib.generators.toPretty {} vals}"; } -- cgit 1.4.1 From e30f48be948272df2d57237ef955023f937f4421 Mon Sep 17 00:00:00 2001 From: Anthony Roussel Date: Mon, 30 Oct 2023 21:41:44 +0100 Subject: treewide: fix redirected and broken URLs Using the script in maintainers/scripts/update-redirected-urls.sh --- CONTRIBUTING.md | 2 +- doc/languages-frameworks/lisp.section.md | 2 +- doc/packages/steam.section.md | 2 +- doc/using/overlays.chapter.md | 2 +- lib/licenses.nix | 10 +++++----- .../haskell/upload-nixos-package-list-to-hackage.sh | 2 +- nixos/doc/manual/configuration/subversion.chapter.md | 4 ++-- .../manual/development/writing-documentation.chapter.md | 8 ++++---- nixos/modules/config/users-groups.nix | 2 +- nixos/modules/installer/tools/nixos-generate-config.pl | 2 +- nixos/modules/programs/npm.nix | 2 +- nixos/modules/programs/zsh/oh-my-zsh.md | 2 +- nixos/modules/security/pam_mount.nix | 6 +++--- nixos/modules/security/polkit.nix | 3 +-- nixos/modules/services/audio/jack.nix | 2 +- nixos/modules/services/computing/boinc/client.nix | 2 +- nixos/modules/services/computing/slurm/slurm.nix | 2 +- nixos/modules/services/databases/couchdb.nix | 2 +- nixos/modules/services/databases/firebird.nix | 2 +- nixos/modules/services/databases/pgmanage.nix | 2 +- nixos/modules/services/databases/postgresql.md | 2 +- nixos/modules/services/databases/postgresql.nix | 2 +- nixos/modules/services/databases/redis.nix | 2 +- nixos/modules/services/hardware/udev.nix | 2 +- nixos/modules/services/logging/logstash.nix | 2 +- nixos/modules/services/misc/mediatomb.nix | 2 +- nixos/modules/services/monitoring/bosun.nix | 2 +- nixos/modules/services/monitoring/grafana.nix | 4 ++-- nixos/modules/services/monitoring/graphite.nix | 2 +- nixos/modules/services/monitoring/munin.nix | 14 +++++++------- nixos/modules/services/monitoring/nagios.nix | 2 +- nixos/modules/services/network-filesystems/xtreemfs.nix | 10 +++++----- nixos/modules/services/networking/asterisk.nix | 2 +- nixos/modules/services/networking/i2pd.nix | 2 +- nixos/modules/services/networking/ntp/ntpd.nix | 2 +- nixos/modules/services/networking/tox-bootstrapd.nix | 2 +- nixos/modules/services/torrent/transmission.nix | 2 +- nixos/modules/services/video/unifi-video.nix | 4 ++-- nixos/modules/services/web-apps/discourse.nix | 4 ++-- nixos/modules/services/web-apps/nextcloud.md | 2 +- nixos/modules/services/web-apps/tt-rss.nix | 2 +- nixos/modules/services/web-servers/lighttpd/default.nix | 6 +++--- nixos/modules/services/web-servers/mighttpd2.nix | 4 ++-- nixos/modules/services/web-servers/nginx/default.nix | 2 +- .../services/x11/desktop-managers/enlightenment.nix | 2 +- nixos/modules/services/x11/desktop-managers/gnome.nix | 2 +- nixos/modules/services/x11/desktop-managers/pantheon.nix | 2 +- nixos/modules/services/x11/display-managers/default.nix | 2 +- nixos/modules/services/x11/display-managers/xpra.nix | 6 +++--- nixos/modules/services/x11/imwheel.nix | 2 +- .../boot/loader/generic-extlinux-compatible/default.nix | 2 +- nixos/modules/system/boot/loader/grub/grub.nix | 2 +- nixos/modules/system/boot/luksroot.nix | 2 +- nixos/modules/system/boot/stage-1.nix | 2 +- pkgs/applications/emulators/gxemul/default.nix | 4 ++-- pkgs/applications/file-managers/dfilemanager/default.nix | 2 +- pkgs/applications/graphics/openscad/default.nix | 2 +- pkgs/applications/graphics/pikopixel/default.nix | 6 +++--- pkgs/applications/graphics/viewnior/default.nix | 2 +- pkgs/applications/misc/yate/default.nix | 2 +- pkgs/applications/networking/irc/kirc/default.nix | 2 +- pkgs/applications/networking/irc/kvirc/default.nix | 2 +- pkgs/applications/networking/p2p/mldonkey/default.nix | 2 +- pkgs/applications/radio/ebook2cw/default.nix | 2 +- pkgs/applications/science/logic/aiger/default.nix | 4 ++-- pkgs/applications/science/logic/cadical/default.nix | 2 +- pkgs/applications/science/logic/cvc3/default.nix | 6 +++--- pkgs/applications/science/logic/kissat/default.nix | 2 +- pkgs/applications/science/logic/picosat/default.nix | 4 ++-- pkgs/applications/science/math/colpack/default.nix | 2 +- pkgs/applications/virtualization/qemu/default.nix | 7 ++----- pkgs/applications/virtualization/virt-manager/default.nix | 2 +- pkgs/applications/virtualization/virt-viewer/default.nix | 2 +- pkgs/data/soundfonts/generaluser/default.nix | 4 ++-- pkgs/development/compilers/pforth/default.nix | 2 +- pkgs/development/compilers/uasm/default.nix | 2 +- pkgs/development/interpreters/picolisp/default.nix | 2 +- pkgs/development/libraries/ctpp2/default.nix | 4 ++-- pkgs/development/libraries/dbus/default.nix | 2 +- pkgs/development/libraries/irrlicht/default.nix | 2 +- pkgs/development/libraries/irrlicht/mac.nix | 2 +- pkgs/development/libraries/libcdr/default.nix | 2 +- pkgs/development/libraries/libevdev/default.nix | 2 +- pkgs/development/libraries/libproxy/default.nix | 2 +- pkgs/development/libraries/libsodium/default.nix | 2 +- pkgs/development/libraries/libtsm/default.nix | 2 +- pkgs/development/libraries/openfx/default.nix | 4 ++-- pkgs/development/libraries/polkit/default.nix | 2 +- pkgs/development/libraries/portaudio/default.nix | 4 ++-- .../development/libraries/startup-notification/default.nix | 2 +- pkgs/development/libraries/vcg/default.nix | 2 +- .../examples/bordeaux-threads.nix | 2 +- pkgs/development/python-modules/cvxopt/default.nix | 2 +- pkgs/development/python-modules/polling/default.nix | 2 +- pkgs/development/python-modules/sqlobject/default.nix | 2 +- pkgs/development/tools/nailgun/default.nix | 2 +- pkgs/games/hedgewars/default.nix | 2 +- pkgs/games/hyperrogue/default.nix | 2 +- pkgs/games/xskat/default.nix | 4 ++-- pkgs/misc/cups/cups-pk-helper.nix | 2 +- pkgs/os-specific/linux/ch9344/default.nix | 2 +- pkgs/os-specific/linux/kmscon/default.nix | 2 +- pkgs/os-specific/linux/microcode/intel.nix | 2 +- pkgs/tools/filesystems/unionfs-fuse/default.nix | 2 +- pkgs/tools/misc/ccal/default.nix | 5 ++--- pkgs/tools/misc/desktop-file-utils/default.nix | 2 +- pkgs/tools/misc/plantuml/default.nix | 2 +- pkgs/tools/networking/redsocks/default.nix | 2 +- pkgs/tools/networking/ripmime/default.nix | 6 +++--- pkgs/tools/security/afl/qemu.nix | 4 ++-- pkgs/tools/security/chkrootkit/default.nix | 4 ++-- 111 files changed, 157 insertions(+), 162 deletions(-) (limited to 'lib') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 483267935c8..dadda215b0d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -353,7 +353,7 @@ In a case a contributor definitively leaves the Nix community, they should creat # Flow of merged pull requests After a pull request is merged, it eventually makes it to the [official Hydra CI](https://hydra.nixos.org/). -Hydra regularly evaluates and builds Nixpkgs, updating [the official channels](http://channels.nixos.org/) when specific Hydra jobs succeeded. +Hydra regularly evaluates and builds Nixpkgs, updating [the official channels](https://channels.nixos.org/) when specific Hydra jobs succeeded. See [Nix Channel Status](https://status.nixos.org/) for the current channels and their state. Here's a brief overview of the main Git branches and what channels they're used for: diff --git a/doc/languages-frameworks/lisp.section.md b/doc/languages-frameworks/lisp.section.md index fe7f2ef80a2..09193093b08 100644 --- a/doc/languages-frameworks/lisp.section.md +++ b/doc/languages-frameworks/lisp.section.md @@ -66,7 +66,7 @@ buildPhase = '' To save some work of writing Nix expressions, there is a script that imports all the packages distributed by Quicklisp into `imported.nix`. This works by parsing its `releases.txt` and `systems.txt` files, which are published every couple of -months on [quicklisp.org](http://beta.quicklisp.org/dist/quicklisp.txt). +months on [quicklisp.org](https://beta.quicklisp.org/dist/quicklisp.txt). The import process is implemented in the `import` directory as Common Lisp code in the `org.lispbuilds.nix` ASDF system. To run the script, one can diff --git a/doc/packages/steam.section.md b/doc/packages/steam.section.md index 25728aa52ae..a1e88b0d971 100644 --- a/doc/packages/steam.section.md +++ b/doc/packages/steam.section.md @@ -11,7 +11,7 @@ Nix problems and constraints: - The `steam.sh` script in `$HOME` cannot be patched, as it is checked and rewritten by steam. - The steam binary cannot be patched, it's also checked. -The current approach to deploy Steam in NixOS is composing a FHS-compatible chroot environment, as documented [here](http://sandervanderburg.blogspot.nl/2013/09/composing-fhs-compatible-chroot.html). This allows us to have binaries in the expected paths without disrupting the system, and to avoid patching them to work in a non FHS environment. +The current approach to deploy Steam in NixOS is composing a FHS-compatible chroot environment, as documented [here](https://sandervanderburg.blogspot.com/2013/09/composing-fhs-compatible-chroot.html). This allows us to have binaries in the expected paths without disrupting the system, and to avoid patching them to work in a non FHS environment. ## How to play {#sec-steam-play} diff --git a/doc/using/overlays.chapter.md b/doc/using/overlays.chapter.md index 1e965e5f0c7..1bec6586f28 100644 --- a/doc/using/overlays.chapter.md +++ b/doc/using/overlays.chapter.md @@ -77,7 +77,7 @@ In Nixpkgs, we have multiple implementations of the BLAS/LAPACK numerical linear The Nixpkgs attribute is `openblas` for ILP64 (integer width = 64 bits) and `openblasCompat` for LP64 (integer width = 32 bits). `openblasCompat` is the default. -- [LAPACK reference](http://www.netlib.org/lapack/) (also provides BLAS and CBLAS) +- [LAPACK reference](https://www.netlib.org/lapack/) (also provides BLAS and CBLAS) The Nixpkgs attribute is `lapack-reference`. diff --git a/lib/licenses.nix b/lib/licenses.nix index a90bab0b15d..ad6922498ab 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -516,17 +516,17 @@ in mkLicense lset) ({ generaluser = { fullName = "GeneralUser GS License v2.0"; - url = "http://www.schristiancollins.com/generaluser.php"; # license included in sources + url = "https://www.schristiancollins.com/generaluser.php"; # license included in sources }; gfl = { fullName = "GUST Font License"; - url = "http://www.gust.org.pl/fonts/licenses/GUST-FONT-LICENSE.txt"; + url = "https://www.gust.org.pl/projects/e-foundry/licenses/GUST-FONT-LICENSE.txt"; }; gfsl = { fullName = "GUST Font Source License"; - url = "http://www.gust.org.pl/fonts/licenses/GUST-FONT-SOURCE-LICENSE.txt"; + url = "https://www.gust.org.pl/projects/e-foundry/licenses/GUST-FONT-SOURCE-LICENSE.txt"; }; gpl1Only = { @@ -613,7 +613,7 @@ in mkLicense lset) ({ info-zip = { spdxId = "Info-ZIP"; fullName = "Info-ZIP License"; - url = "http://www.info-zip.org/pub/infozip/license.html"; + url = "https://infozip.sourceforge.net/license.html"; }; inria-compcert = { @@ -1182,7 +1182,7 @@ in mkLicense lset) ({ xfig = { fullName = "xfig"; - url = "http://mcj.sourceforge.net/authors.html#xfig"; # https is broken + url = "https://mcj.sourceforge.net/authors.html#xfig"; }; zlib = { diff --git a/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh index 86fecbc3d87..9130941a536 100755 --- a/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh +++ b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh @@ -39,5 +39,5 @@ fi package_list="$(nix-build -A haskell.package-list)/nixos-hackage-packages.csv" username=$(grep "^username:" "$CABAL_DIR/config" | sed "s/^username: //") password_command=$(grep "^password-command:" "$CABAL_DIR/config" | sed "s/^password-command: //") -curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" http://hackage.haskell.org/distro/NixOS/packages.csv +curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" https://hackage.haskell.org/distro/NixOS/packages.csv echo diff --git a/nixos/doc/manual/configuration/subversion.chapter.md b/nixos/doc/manual/configuration/subversion.chapter.md index 84f9c270337..ff870f5c40b 100644 --- a/nixos/doc/manual/configuration/subversion.chapter.md +++ b/nixos/doc/manual/configuration/subversion.chapter.md @@ -2,7 +2,7 @@ [Subversion](https://subversion.apache.org/) is a centralized version-control system. It can use a [variety of -protocols](http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.serverconfig.choosing) +protocols](https://svnbook.red-bean.com/en/1.7/svn-book.html#svn.serverconfig.choosing) for communication between client and server. ## Subversion inside Apache HTTP {#module-services-subversion-apache-httpd} @@ -14,7 +14,7 @@ for communication. For more information on the general setup, please refer to the [the appropriate section of the Subversion -book](http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.serverconfig.httpd). +book](https://svnbook.red-bean.com/en/1.7/svn-book.html#svn.serverconfig.httpd). To configure, include in `/etc/nixos/configuration.nix` code to activate Apache HTTP, setting [](#opt-services.httpd.adminAddr) diff --git a/nixos/doc/manual/development/writing-documentation.chapter.md b/nixos/doc/manual/development/writing-documentation.chapter.md index c07a2618c07..3d9bd318cf3 100644 --- a/nixos/doc/manual/development/writing-documentation.chapter.md +++ b/nixos/doc/manual/development/writing-documentation.chapter.md @@ -33,13 +33,13 @@ symlink at `./result/share/doc/nixos/index.html`. ## Editing DocBook XML {#sec-writing-docs-editing-docbook-xml} For general information on how to write in DocBook, see [DocBook 5: The -Definitive Guide](http://www.docbook.org/tdg5/en/html/docbook.html). +Definitive Guide](https://tdg.docbook.org/tdg/5.1/). Emacs nXML Mode is very helpful for editing DocBook XML because it validates the document as you write, and precisely locates errors. To use it, see [](#sec-emacs-docbook-xml). -[Pandoc](http://pandoc.org) can generate DocBook XML from a multitude of +[Pandoc](https://pandoc.org/) can generate DocBook XML from a multitude of formats, which makes a good starting point. Here is an example of Pandoc invocation to convert GitHub-Flavoured MarkDown to DocBook 5 XML: @@ -62,9 +62,9 @@ topic from scratch. Keep the following guidelines in mind when you create and add a topic: -- The NixOS [`book`](http://www.docbook.org/tdg5/en/html/book.html) +- The NixOS [`book`](https://tdg.docbook.org/tdg/5.0/book.html) element is in `nixos/doc/manual/manual.xml`. It includes several - [`parts`](http://www.docbook.org/tdg5/en/html/book.html) which are in + [`parts`](https://tdg.docbook.org/tdg/5.0/book.html) which are in subdirectories. - Store the topic file in the same directory as the `part` to which it diff --git a/nixos/modules/config/users-groups.nix b/nixos/modules/config/users-groups.nix index b4251214876..39aac9fb821 100644 --- a/nixos/modules/config/users-groups.nix +++ b/nixos/modules/config/users-groups.nix @@ -153,7 +153,7 @@ let {file}`pam_mount.conf.xml`. Useful attributes might include `path`, `options`, `fstype`, and `server`. - See + See for more information. ''; }; diff --git a/nixos/modules/installer/tools/nixos-generate-config.pl b/nixos/modules/installer/tools/nixos-generate-config.pl index 85180bf2d1b..71737cd8ebc 100644 --- a/nixos/modules/installer/tools/nixos-generate-config.pl +++ b/nixos/modules/installer/tools/nixos-generate-config.pl @@ -130,7 +130,7 @@ sub pciCheck { debug "\n"; if (defined $module) { - # See the bottom of http://pciids.sourceforge.net/pci.ids for + # See the bottom of https://pciids.sourceforge.net/pci.ids for # device classes. if (# Mass-storage controller. Definitely important. $class =~ /^0x01/ || diff --git a/nixos/modules/programs/npm.nix b/nixos/modules/programs/npm.nix index 48dc48e668f..c41fea32614 100644 --- a/nixos/modules/programs/npm.nix +++ b/nixos/modules/programs/npm.nix @@ -34,7 +34,7 @@ in prefix = ''${HOME}/.npm https-proxy=proxy.example.com init-license=MIT - init-author-url=http://npmjs.org + init-author-url=https://www.npmjs.com/ color=true ''; }; diff --git a/nixos/modules/programs/zsh/oh-my-zsh.md b/nixos/modules/programs/zsh/oh-my-zsh.md index 73d425244ce..6a310006edb 100644 --- a/nixos/modules/programs/zsh/oh-my-zsh.md +++ b/nixos/modules/programs/zsh/oh-my-zsh.md @@ -78,7 +78,7 @@ If third-party customizations (e.g. new themes) are supposed to be added to - Completion scripts are supposed to be stored at `$out/share/zsh/site-functions`. This directory is part of the - [`fpath`](http://zsh.sourceforge.net/Doc/Release/Functions.html) + [`fpath`](https://zsh.sourceforge.io/Doc/Release/Functions.html) and the package should be compatible with pure `ZSH` setups. The module will automatically link the contents of `site-functions` to completions directory in the proper diff --git a/nixos/modules/security/pam_mount.nix b/nixos/modules/security/pam_mount.nix index ad78f38b086..26f906f2a76 100644 --- a/nixos/modules/security/pam_mount.nix +++ b/nixos/modules/security/pam_mount.nix @@ -33,7 +33,7 @@ in default = []; description = lib.mdDoc '' List of volume definitions for pam_mount. - For more information, visit . + For more information, visit . ''; }; @@ -78,7 +78,7 @@ in description = lib.mdDoc '' Sets the Debug-Level. 0 disables debugging, 1 enables pam_mount tracing, and 2 additionally enables tracing in mount.crypt. The default is 0. - For more information, visit . + For more information, visit . ''; }; @@ -88,7 +88,7 @@ in description = lib.mdDoc '' Amount of microseconds to wait until killing remaining processes after final logout. - For more information, visit . + For more information, visit . ''; }; diff --git a/nixos/modules/security/polkit.nix b/nixos/modules/security/polkit.nix index de427ccb295..327f49c0b63 100644 --- a/nixos/modules/security/polkit.nix +++ b/nixos/modules/security/polkit.nix @@ -35,7 +35,7 @@ in description = lib.mdDoc '' Any polkit rules to be added to config (in JavaScript ;-). See: - http://www.freedesktop.org/software/polkit/docs/latest/polkit.8.html#polkit-rules + ''; }; @@ -117,4 +117,3 @@ in }; } - diff --git a/nixos/modules/services/audio/jack.nix b/nixos/modules/services/audio/jack.nix index 105e99cb2f5..b51f2a78c98 100644 --- a/nixos/modules/services/audio/jack.nix +++ b/nixos/modules/services/audio/jack.nix @@ -225,7 +225,7 @@ in { description = "JACK Audio system service user"; isSystemUser = true; }; - # http://jackaudio.org/faq/linux_rt_config.html + # https://jackaudio.org/faq/linux_rt_config.html security.pam.loginLimits = [ { domain = "@jackaudio"; type = "-"; item = "rtprio"; value = "99"; } { domain = "@jackaudio"; type = "-"; item = "memlock"; value = "unlimited"; } diff --git a/nixos/modules/services/computing/boinc/client.nix b/nixos/modules/services/computing/boinc/client.nix index 51475171bf3..ff16795c820 100644 --- a/nixos/modules/services/computing/boinc/client.nix +++ b/nixos/modules/services/computing/boinc/client.nix @@ -54,7 +54,7 @@ in only the hosts listed in {var}`dataDir`/remote_hosts.cfg will be allowed to connect. - See also: + See also: ''; }; diff --git a/nixos/modules/services/computing/slurm/slurm.nix b/nixos/modules/services/computing/slurm/slurm.nix index 344c43a429b..1cbe7b893f8 100644 --- a/nixos/modules/services/computing/slurm/slurm.nix +++ b/nixos/modules/services/computing/slurm/slurm.nix @@ -6,7 +6,7 @@ let cfg = config.services.slurm; opt = options.services.slurm; - # configuration file can be generated by http://slurm.schedmd.com/configurator.html + # configuration file can be generated by https://slurm.schedmd.com/configurator.html defaultUser = "slurm"; diff --git a/nixos/modules/services/databases/couchdb.nix b/nixos/modules/services/databases/couchdb.nix index 0a81a8dceee..bfecfbb3664 100644 --- a/nixos/modules/services/databases/couchdb.nix +++ b/nixos/modules/services/databases/couchdb.nix @@ -79,7 +79,7 @@ in { ''; }; - # couchdb options: http://docs.couchdb.org/en/latest/config/index.html + # couchdb options: https://docs.couchdb.org/en/latest/config/index.html databaseDir = mkOption { type = types.path; diff --git a/nixos/modules/services/databases/firebird.nix b/nixos/modules/services/databases/firebird.nix index 26ed46f0e60..3927c81d953 100644 --- a/nixos/modules/services/databases/firebird.nix +++ b/nixos/modules/services/databases/firebird.nix @@ -17,7 +17,7 @@ # There are at least two ways to run firebird. superserver has been chosen # however there are no strong reasons to prefer this or the other one AFAIK # Eg superserver is said to be most efficiently using resources according to -# http://www.firebirdsql.org/manual/qsg25-classic-or-super.html +# https://www.firebirdsql.org/manual/qsg25-classic-or-super.html with lib; diff --git a/nixos/modules/services/databases/pgmanage.nix b/nixos/modules/services/databases/pgmanage.nix index 12c8253ab49..a0933a5ffc4 100644 --- a/nixos/modules/services/databases/pgmanage.nix +++ b/nixos/modules/services/databases/pgmanage.nix @@ -66,7 +66,7 @@ in { pgmanage requires at least one PostgreSQL server be defined. Detailed information about PostgreSQL connection strings is available at: - + Note that you should not specify your user name or password. That information will be entered on the login screen. If you specify a diff --git a/nixos/modules/services/databases/postgresql.md b/nixos/modules/services/databases/postgresql.md index e4b679a3eee..d65d9616e2f 100644 --- a/nixos/modules/services/databases/postgresql.md +++ b/nixos/modules/services/databases/postgresql.md @@ -5,7 +5,7 @@ *Source:* {file}`modules/services/databases/postgresql.nix` -*Upstream documentation:* +*Upstream documentation:* diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 21e6a60e32a..af4db5c9611 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -588,7 +588,7 @@ in else "simple"; # Shut down Postgres using SIGINT ("Fast Shutdown mode"). See - # http://www.postgresql.org/docs/current/static/server-shutdown.html + # https://www.postgresql.org/docs/current/server-shutdown.html KillSignal = "SIGINT"; KillMode = "mixed"; diff --git a/nixos/modules/services/databases/redis.nix b/nixos/modules/services/databases/redis.nix index 86b295dadf4..315a0282cd7 100644 --- a/nixos/modules/services/databases/redis.nix +++ b/nixos/modules/services/databases/redis.nix @@ -63,7 +63,7 @@ in { vmOverCommit = mkEnableOption (lib.mdDoc '' setting of vm.overcommit_memory to 1 - (Suggested for Background Saving: http://redis.io/topics/faq) + (Suggested for Background Saving: ) ''); servers = mkOption { diff --git a/nixos/modules/services/hardware/udev.nix b/nixos/modules/services/hardware/udev.nix index 24987374ab0..08ca7a0d247 100644 --- a/nixos/modules/services/hardware/udev.nix +++ b/nixos/modules/services/hardware/udev.nix @@ -279,7 +279,7 @@ in default = true; type = types.bool; description = lib.mdDoc '' - Whether to assign [predictable names to network interfaces](http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames). + Whether to assign [predictable names to network interfaces](https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/). If enabled, interfaces are assigned names that contain topology information (e.g. `wlp3s0`) and thus should be stable diff --git a/nixos/modules/services/logging/logstash.nix b/nixos/modules/services/logging/logstash.nix index e9e3ae1f14c..42d52a61639 100644 --- a/nixos/modules/services/logging/logstash.nix +++ b/nixos/modules/services/logging/logstash.nix @@ -123,7 +123,7 @@ in example = '' if [type] == "syslog" { # Keep only relevant systemd fields - # http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html + # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html prune { whitelist_names => [ "type", "@timestamp", "@version", diff --git a/nixos/modules/services/misc/mediatomb.nix b/nixos/modules/services/misc/mediatomb.nix index 632b7caaac4..335b1b684b1 100644 --- a/nixos/modules/services/misc/mediatomb.nix +++ b/nixos/modules/services/misc/mediatomb.nix @@ -186,7 +186,7 @@ let defaultFirewallRules = { # udp 1900 port needs to be opened for SSDP (not configurable within # mediatomb/gerbera) cf. - # http://docs.gerbera.io/en/latest/run.html?highlight=udp%20port#network-setup + # https://docs.gerbera.io/en/latest/run.html?highlight=udp%20port#network-setup allowedUDPPorts = [ 1900 cfg.port ]; allowedTCPPorts = [ cfg.port ]; }; diff --git a/nixos/modules/services/monitoring/bosun.nix b/nixos/modules/services/monitoring/bosun.nix index dc75fda6ed8..1dc19743461 100644 --- a/nixos/modules/services/monitoring/bosun.nix +++ b/nixos/modules/services/monitoring/bosun.nix @@ -108,7 +108,7 @@ in { option. A detailed description of the supported syntax can be found at-spi2-atk - http://bosun.org/configuration.html + https://bosun.org/configuration.html ''; }; diff --git a/nixos/modules/services/monitoring/grafana.nix b/nixos/modules/services/monitoring/grafana.nix index e90a0e9d16d..5e21407042b 100644 --- a/nixos/modules/services/monitoring/grafana.nix +++ b/nixos/modules/services/monitoring/grafana.nix @@ -88,7 +88,7 @@ let # Get a submodule without any embedded metadata: _filter = x: filterAttrs (k: v: k != "_module") x; - # http://docs.grafana.org/administration/provisioning/#datasources + # https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources grafanaTypes.datasourceConfig = types.submodule { freeformType = provisioningSettingsFormat.type; @@ -140,7 +140,7 @@ let }; }; - # http://docs.grafana.org/administration/provisioning/#dashboards + # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards grafanaTypes.dashboardConfig = types.submodule { freeformType = provisioningSettingsFormat.type; diff --git a/nixos/modules/services/monitoring/graphite.nix b/nixos/modules/services/monitoring/graphite.nix index 65c91b8f79b..cc3d7097620 100644 --- a/nixos/modules/services/monitoring/graphite.nix +++ b/nixos/modules/services/monitoring/graphite.nix @@ -102,7 +102,7 @@ in { default = ""; description = lib.mdDoc '' Graphite webapp settings. See: - + ''; }; }; diff --git a/nixos/modules/services/monitoring/munin.nix b/nixos/modules/services/monitoring/munin.nix index 90a51181ac3..5ed7cac48ae 100644 --- a/nixos/modules/services/monitoring/munin.nix +++ b/nixos/modules/services/monitoring/munin.nix @@ -4,7 +4,7 @@ # TODO: LWP/Pg perl libs aren't recognized # TODO: support fastcgi -# http://guide.munin-monitoring.org/en/latest/example/webserver/apache-cgi.html +# https://guide.munin-monitoring.org/en/latest/example/webserver/apache-cgi.html # spawn-fcgi -s /run/munin/fastcgi-graph.sock -U www-data -u munin -g munin /usr/lib/munin/cgi/munin-cgi-graph # spawn-fcgi -s /run/munin/fastcgi-html.sock -U www-data -u munin -g munin /usr/lib/munin/cgi/munin-cgi-html # https://paste.sh/vofcctHP#-KbDSXVeWoifYncZmLfZzgum @@ -147,7 +147,7 @@ in Enable Munin Node agent. Munin node listens on 0.0.0.0 and by default accepts connections only from 127.0.0.1 for security reasons. - See . + See . ''; }; @@ -156,7 +156,7 @@ in type = types.lines; description = lib.mdDoc '' {file}`munin-node.conf` extra configuration. See - + ''; }; @@ -165,7 +165,7 @@ in type = types.lines; description = lib.mdDoc '' {file}`plugin-conf.d` extra plugin configuration. See - + ''; example = '' [fail2ban_*] @@ -273,9 +273,9 @@ in type = types.lines; description = lib.mdDoc '' {file}`munin.conf` extra global configuration. - See . + See . Useful to setup notifications, see - + ''; example = '' contact.email.command mail -s "Munin notification for ''${var:host}" someone@example.com @@ -288,7 +288,7 @@ in description = lib.mdDoc '' Definitions of hosts of nodes to collect data from. Needs at least one host for cron to succeed. See - + ''; example = literalExpression '' ''' diff --git a/nixos/modules/services/monitoring/nagios.nix b/nixos/modules/services/monitoring/nagios.nix index 8feff22c118..dc5fa1be292 100644 --- a/nixos/modules/services/monitoring/nagios.nix +++ b/nixos/modules/services/monitoring/nagios.nix @@ -88,7 +88,7 @@ in options = { services.nagios = { - enable = mkEnableOption (lib.mdDoc ''[Nagios](http://www.nagios.org/) to monitor your system or network.''); + enable = mkEnableOption (lib.mdDoc ''[Nagios](https://www.nagios.org/) to monitor your system or network.''); objectDefs = mkOption { description = lib.mdDoc '' diff --git a/nixos/modules/services/network-filesystems/xtreemfs.nix b/nixos/modules/services/network-filesystems/xtreemfs.nix index 926c3c3bd52..866661cf4e6 100644 --- a/nixos/modules/services/network-filesystems/xtreemfs.nix +++ b/nixos/modules/services/network-filesystems/xtreemfs.nix @@ -176,7 +176,7 @@ in description = lib.mdDoc '' Configuration of XtreemFS DIR service. WARNING: configuration is saved as plaintext inside nix store. - For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html + For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html ''; }; replication = { @@ -218,7 +218,7 @@ in description = lib.mdDoc '' Configuration of XtreemFS DIR replication plugin. WARNING: configuration is saved as plaintext inside nix store. - For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html + For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html ''; }; }; @@ -319,7 +319,7 @@ in description = lib.mdDoc '' Configuration of XtreemFS MRC service. WARNING: configuration is saved as plaintext inside nix store. - For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html + For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html ''; }; replication = { @@ -361,7 +361,7 @@ in description = lib.mdDoc '' Configuration of XtreemFS MRC replication plugin. WARNING: configuration is saved as plaintext inside nix store. - For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html + For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html ''; }; }; @@ -438,7 +438,7 @@ in description = lib.mdDoc '' Configuration of XtreemFS OSD service. WARNING: configuration is saved as plaintext inside nix store. - For more options: http://www.xtreemfs.org/xtfs-guide-1.5.1/index.html + For more options: https://www.xtreemfs.org/xtfs-guide-1.5.1/index.html ''; }; }; diff --git a/nixos/modules/services/networking/asterisk.nix b/nixos/modules/services/networking/asterisk.nix index 5a1d03f0721..279927781ed 100644 --- a/nixos/modules/services/networking/asterisk.nix +++ b/nixos/modules/services/networking/asterisk.nix @@ -139,7 +139,7 @@ in path. See - + for more examples of what is possible here. ''; }; diff --git a/nixos/modules/services/networking/i2pd.nix b/nixos/modules/services/networking/i2pd.nix index f872daf05b8..808e7b66d36 100644 --- a/nixos/modules/services/networking/i2pd.nix +++ b/nixos/modules/services/networking/i2pd.nix @@ -239,7 +239,7 @@ in enable = mkEnableOption (lib.mdDoc "I2Pd daemon") // { description = lib.mdDoc '' Enables I2Pd as a running service upon activation. - Please read http://i2pd.readthedocs.io/en/latest/ for further + Please read for further configuration help. ''; }; diff --git a/nixos/modules/services/networking/ntp/ntpd.nix b/nixos/modules/services/networking/ntp/ntpd.nix index 036a8df635d..2bc690cacf0 100644 --- a/nixos/modules/services/networking/ntp/ntpd.nix +++ b/nixos/modules/services/networking/ntp/ntpd.nix @@ -56,7 +56,7 @@ in The default flags prevent external hosts from using ntpd as a DDoS reflector, setting system time, and querying OS/ntpd version. As recommended in section 6.5.1.1.3, answer "No" of - http://support.ntp.org/bin/view/Support/AccessRestrictions + https://support.ntp.org/Support/AccessRestrictions ''; default = [ "limited" "kod" "nomodify" "notrap" "noquery" "nopeer" ]; }; diff --git a/nixos/modules/services/networking/tox-bootstrapd.nix b/nixos/modules/services/networking/tox-bootstrapd.nix index 5c7e7a4c220..0f310a28d26 100644 --- a/nixos/modules/services/networking/tox-bootstrapd.nix +++ b/nixos/modules/services/networking/tox-bootstrapd.nix @@ -47,7 +47,7 @@ in lib.mdDoc '' Configuration for bootstrap daemon. See - and . + and . ''; }; }; diff --git a/nixos/modules/services/torrent/transmission.nix b/nixos/modules/services/torrent/transmission.nix index b98cb5283a1..5efb9334ea0 100644 --- a/nixos/modules/services/torrent/transmission.nix +++ b/nixos/modules/services/torrent/transmission.nix @@ -148,7 +148,7 @@ in type = types.bool; default = true; description = lib.mdDoc '' - Whether to enable [Micro Transport Protocol (µTP)](http://en.wikipedia.org/wiki/Micro_Transport_Protocol). + Whether to enable [Micro Transport Protocol (µTP)](https://en.wikipedia.org/wiki/Micro_Transport_Protocol). ''; }; options.watch-dir = mkOption { diff --git a/nixos/modules/services/video/unifi-video.nix b/nixos/modules/services/video/unifi-video.nix index cb438a08150..5c93f60cbd7 100644 --- a/nixos/modules/services/video/unifi-video.nix +++ b/nixos/modules/services/video/unifi-video.nix @@ -32,7 +32,7 @@ let name = "mongo.conf"; executable = false; text = '' - # for documentation of all options, see http://docs.mongodb.org/manual/reference/configuration-options/ + # for documentation of all options, see https://www.mongodb.com/docs/manual/reference/configuration-options/ storage: dbPath: ${cfg.dataDir}/db @@ -63,7 +63,7 @@ let executable = false; text = '' # for documentation of all options, see: - # http://docs.mongodb.org/manual/reference/configuration-options/ + # https://www.mongodb.com/docs/manual/reference/configuration-options/ storage: dbPath: ${cfg.dataDir}/db-wt diff --git a/nixos/modules/services/web-apps/discourse.nix b/nixos/modules/services/web-apps/discourse.nix index f80eb6b4c7f..da1dba7d940 100644 --- a/nixos/modules/services/web-apps/discourse.nix +++ b/nixos/modules/services/web-apps/discourse.nix @@ -407,7 +407,7 @@ in type = with lib.types; nullOr (enum ["plain" "login" "cram_md5"]); default = null; description = lib.mdDoc '' - Authentication type to use, see http://api.rubyonrails.org/classes/ActionMailer/Base.html + Authentication type to use, see https://api.rubyonrails.org/classes/ActionMailer/Base.html ''; }; @@ -423,7 +423,7 @@ in type = lib.types.str; default = "peer"; description = lib.mdDoc '' - How OpenSSL checks the certificate, see http://api.rubyonrails.org/classes/ActionMailer/Base.html + How OpenSSL checks the certificate, see https://api.rubyonrails.org/classes/ActionMailer/Base.html ''; }; diff --git a/nixos/modules/services/web-apps/nextcloud.md b/nixos/modules/services/web-apps/nextcloud.md index a25bed30e47..ecc7f380592 100644 --- a/nixos/modules/services/web-apps/nextcloud.md +++ b/nixos/modules/services/web-apps/nextcloud.md @@ -49,7 +49,7 @@ used by the imperative installer and all values are written to an additional fil to ensure that changes can be applied by changing the module's options. In case the application serves multiple domains (those are checked with -[`$_SERVER['HTTP_HOST']`](http://php.net/manual/en/reserved.variables.server.php)) +[`$_SERVER['HTTP_HOST']`](https://www.php.net/manual/en/reserved.variables.server.php)) it's needed to add them to [`services.nextcloud.config.extraTrustedDomains`](#opt-services.nextcloud.config.extraTrustedDomains). diff --git a/nixos/modules/services/web-apps/tt-rss.nix b/nixos/modules/services/web-apps/tt-rss.nix index 592ab253f7d..7b2e3be4295 100644 --- a/nixos/modules/services/web-apps/tt-rss.nix +++ b/nixos/modules/services/web-apps/tt-rss.nix @@ -430,7 +430,7 @@ let background processes while not running tt-rss, this method is generally viable to keep your feeds up to date. Still, there are more robust (and recommended) updating methods - available, you can read about them here: http://tt-rss.org/wiki/UpdatingFeeds + available, you can read about them here: ''; }; diff --git a/nixos/modules/services/web-servers/lighttpd/default.nix b/nixos/modules/services/web-servers/lighttpd/default.nix index 729a633a36c..eaa113c0d52 100644 --- a/nixos/modules/services/web-servers/lighttpd/default.nix +++ b/nixos/modules/services/web-servers/lighttpd/default.nix @@ -10,7 +10,7 @@ let # List of known lighttpd modules, ordered by how the lighttpd documentation # recommends them being imported: - # http://redmine.lighttpd.net/projects/1/wiki/Server_modulesDetails + # https://redmine.lighttpd.net/projects/1/wiki/Server_modulesDetails # # Some modules are always imported and should not appear in the config: # disallowedModules = [ "mod_indexfile" "mod_dirlisting" "mod_staticfile" ]; @@ -84,8 +84,8 @@ let # server.modules += () entries in each sub-service extraConfig snippet, # read this: # - # http://redmine.lighttpd.net/projects/1/wiki/Server_modulesDetails - # http://redmine.lighttpd.net/issues/2337 + # https://redmine.lighttpd.net/projects/1/wiki/Server_modulesDetails + # https://redmine.lighttpd.net/issues/2337 # # Basically, lighttpd doesn't want to load (or even silently ignore) a # module for a second time, and there is no way to check if a module has diff --git a/nixos/modules/services/web-servers/mighttpd2.nix b/nixos/modules/services/web-servers/mighttpd2.nix index 2d887af87c7..bdd6d8b62aa 100644 --- a/nixos/modules/services/web-servers/mighttpd2.nix +++ b/nixos/modules/services/web-servers/mighttpd2.nix @@ -44,7 +44,7 @@ in { type = types.lines; description = lib.mdDoc '' Verbatim config file to use - (see http://www.mew.org/~kazu/proj/mighttpd/en/config.html) + (see https://kazu-yamamoto.github.io/mighttpd2/config.html) ''; }; @@ -78,7 +78,7 @@ in { type = types.lines; description = lib.mdDoc '' Verbatim routing file to use - (see http://www.mew.org/~kazu/proj/mighttpd/en/config.html) + (see https://kazu-yamamoto.github.io/mighttpd2/config.html) ''; }; diff --git a/nixos/modules/services/web-servers/nginx/default.nix b/nixos/modules/services/web-servers/nginx/default.nix index f2e8585a936..cf70dc32594 100644 --- a/nixos/modules/services/web-servers/nginx/default.nix +++ b/nixos/modules/services/web-servers/nginx/default.nix @@ -955,7 +955,7 @@ in default = {}; description = lib.mdDoc '' Configure a proxy cache path entry. - See for documentation. + See for documentation. ''; }; diff --git a/nixos/modules/services/x11/desktop-managers/enlightenment.nix b/nixos/modules/services/x11/desktop-managers/enlightenment.nix index d4b2a50cb8a..28dd408c923 100644 --- a/nixos/modules/services/x11/desktop-managers/enlightenment.nix +++ b/nixos/modules/services/x11/desktop-managers/enlightenment.nix @@ -63,7 +63,7 @@ in # make available for D-BUS user services #export XDG_DATA_DIRS=$XDG_DATA_DIRS''${XDG_DATA_DIRS:+:}:${config.system.path}/share:${e.efl}/share - # Update user dirs as described in http://freedesktop.org/wiki/Software/xdg-user-dirs/ + # Update user dirs as described in https://freedesktop.org/wiki/Software/xdg-user-dirs/ ${pkgs.xdg-user-dirs}/bin/xdg-user-dirs-update fi ''; diff --git a/nixos/modules/services/x11/desktop-managers/gnome.nix b/nixos/modules/services/x11/desktop-managers/gnome.nix index 8b5daf83de1..12bdd933337 100644 --- a/nixos/modules/services/x11/desktop-managers/gnome.nix +++ b/nixos/modules/services/x11/desktop-managers/gnome.nix @@ -455,7 +455,7 @@ in pkgs.glib # for gsettings program pkgs.gnome-menus pkgs.gtk3.out # for gtk-launch program - pkgs.xdg-user-dirs # Update user dirs as described in http://freedesktop.org/wiki/Software/xdg-user-dirs/ + pkgs.xdg-user-dirs # Update user dirs as described in https://freedesktop.org/wiki/Software/xdg-user-dirs/ ]; in mandatoryPackages diff --git a/nixos/modules/services/x11/desktop-managers/pantheon.nix b/nixos/modules/services/x11/desktop-managers/pantheon.nix index eef7aa14057..d82d19b26cd 100644 --- a/nixos/modules/services/x11/desktop-managers/pantheon.nix +++ b/nixos/modules/services/x11/desktop-managers/pantheon.nix @@ -201,7 +201,7 @@ in onboard orca # elementary/greeter#668 sound-theme-freedesktop - xdg-user-dirs # Update user dirs as described in http://freedesktop.org/wiki/Software/xdg-user-dirs/ + xdg-user-dirs # Update user dirs as described in https://freedesktop.org/wiki/Software/xdg-user-dirs/ ]) ++ (with pkgs.pantheon; [ # Artwork elementary-gtk-theme diff --git a/nixos/modules/services/x11/display-managers/default.nix b/nixos/modules/services/x11/display-managers/default.nix index 1f08ded7c96..16a7ff1a4bd 100644 --- a/nixos/modules/services/x11/display-managers/default.nix +++ b/nixos/modules/services/x11/display-managers/default.nix @@ -96,7 +96,7 @@ let )} # Speed up application start by 50-150ms according to - # http://kdemonkey.blogspot.nl/2008/04/magic-trick.html + # https://kdemonkey.blogspot.com/2008/04/magic-trick.html compose_cache="''${XCOMPOSECACHE:-$HOME/.compose-cache}" mkdir -p "$compose_cache" # To avoid accidentally deleting a wrongly set up XCOMPOSECACHE directory, diff --git a/nixos/modules/services/x11/display-managers/xpra.nix b/nixos/modules/services/x11/display-managers/xpra.nix index cb78f52d9b6..0861530f21e 100644 --- a/nixos/modules/services/x11/display-managers/xpra.nix +++ b/nixos/modules/services/x11/display-managers/xpra.nix @@ -60,11 +60,11 @@ in VertRefresh 1.0 - 200.0 #To add your own modes here, use a modeline calculator, like: # cvt: - # http://www.x.org/archive/X11R7.5/doc/man/man1/cvt.1.html + # https://www.x.org/archive/X11R7.5/doc/man/man1/cvt.1.html # xtiming: - # http://xtiming.sourceforge.net/cgi-bin/xtiming.pl + # https://xtiming.sourceforge.net/cgi-bin/xtiming.pl # gtf: - # http://gtf.sourceforge.net/ + # https://gtf.sourceforge.net/ #This can be used to get a specific DPI, but only for the default resolution: #DisplaySize 508 317 #NOTE: the highest modes will not work without increasing the VideoRam diff --git a/nixos/modules/services/x11/imwheel.nix b/nixos/modules/services/x11/imwheel.nix index 133e64c65cd..bd2bcb7bcd0 100644 --- a/nixos/modules/services/x11/imwheel.nix +++ b/nixos/modules/services/x11/imwheel.nix @@ -37,7 +37,7 @@ in Window class translation rules. /etc/X11/imwheelrc is generated based on this config which means this config is global for all users. - See [official man pages](http://imwheel.sourceforge.net/imwheel.1.html) + See [official man pages](https://imwheel.sourceforge.net/imwheel.1.html) for more information. ''; }; diff --git a/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix b/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix index 5ef3c5cd52a..13df6090711 100644 --- a/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix +++ b/nixos/modules/system/boot/loader/generic-extlinux-compatible/default.nix @@ -25,7 +25,7 @@ in under `/boot/extlinux.conf`. For instance, U-Boot's generic distro boot support uses this file format. - See [U-boot's documentation](http://git.denx.de/?p=u-boot.git;a=blob;f=doc/README.distro;hb=refs/heads/master) + See [U-boot's documentation](https://u-boot.readthedocs.io/en/latest/develop/distro.html) for more information. ''; }; diff --git a/nixos/modules/system/boot/loader/grub/grub.nix b/nixos/modules/system/boot/loader/grub/grub.nix index 468f701ae5b..7097e1d83dc 100644 --- a/nixos/modules/system/boot/loader/grub/grub.nix +++ b/nixos/modules/system/boot/loader/grub/grub.nix @@ -339,7 +339,7 @@ in See the [ GRUB source code - ](http://git.savannah.gnu.org/cgit/grub.git/tree/grub-core/commands/nativedisk.c?h=grub-2.04#n326) + ](https://git.savannah.gnu.org/cgit/grub.git/tree/grub-core/commands/nativedisk.c?h=grub-2.04#n326) for which disk modules are available. The list elements are passed directly as `argv` diff --git a/nixos/modules/system/boot/luksroot.nix b/nixos/modules/system/boot/luksroot.nix index 06c329e006b..ca560d63f3b 100644 --- a/nixos/modules/system/boot/luksroot.nix +++ b/nixos/modules/system/boot/luksroot.nix @@ -537,7 +537,7 @@ in description = lib.mdDoc '' Unless enabled, encryption keys can be easily recovered by an attacker with physical access to any machine with PCMCIA, ExpressCard, ThunderBolt or FireWire port. - More information is available at . + More information is available at . This option blacklists FireWire drivers, but doesn't remove them. You can manually load the drivers if you need to use a FireWire device, but don't forget to unload them! diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 322a234e0c4..13c71e1495e 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -284,7 +284,7 @@ let # in the NixOS installation CD, so use ID_CDROM_MEDIA in the # corresponding udev rules for now. This was the behaviour in # udev <= 154. See also - # http://www.spinics.net/lists/hotplug/msg03935.html + # https://www.spinics.net/lists/hotplug/msg03935.html substituteInPlace $out/60-persistent-storage.rules \ --replace ID_CDROM_MEDIA_TRACK_COUNT_DATA ID_CDROM_MEDIA ''; # */ diff --git a/pkgs/applications/emulators/gxemul/default.nix b/pkgs/applications/emulators/gxemul/default.nix index cacb521c2fb..05f837b1876 100644 --- a/pkgs/applications/emulators/gxemul/default.nix +++ b/pkgs/applications/emulators/gxemul/default.nix @@ -9,7 +9,7 @@ stdenv.mkDerivation rec { version = "0.7.0"; src = fetchurl { - url = "http://gavare.se/gxemul/src/${pname}-${version}.tar.gz"; + url = "https://gavare.se/gxemul/src/${pname}-${version}.tar.gz"; sha256 = "sha256-ecRDfG+MqQT0bTOsNgYqZf3PSpKiSEeOQIqxEpXPjoM="; }; @@ -29,7 +29,7 @@ stdenv.mkDerivation rec { ''; meta = with lib; { - homepage = "http://gavare.se/gxemul/"; + homepage = "https://gavare.se/gxemul/"; description = "Gavare's experimental emulator"; longDescription = '' GXemul is a framework for full-system computer architecture diff --git a/pkgs/applications/file-managers/dfilemanager/default.nix b/pkgs/applications/file-managers/dfilemanager/default.nix index 2c374511af5..6c5544aded3 100644 --- a/pkgs/applications/file-managers/dfilemanager/default.nix +++ b/pkgs/applications/file-managers/dfilemanager/default.nix @@ -17,7 +17,7 @@ mkDerivation { cmakeFlags = [ "-DQT5BUILD=true" ]; meta = { - homepage = "http://dfilemanager.sourceforge.net/"; + homepage = "https://github.com/probonopd/dfilemanager"; description = "File manager written in Qt/C++"; license = lib.licenses.gpl2; platforms = lib.platforms.unix; diff --git a/pkgs/applications/graphics/openscad/default.nix b/pkgs/applications/graphics/openscad/default.nix index 37fcc0eb48e..fc048d4acae 100644 --- a/pkgs/applications/graphics/openscad/default.nix +++ b/pkgs/applications/graphics/openscad/default.nix @@ -105,7 +105,7 @@ mkDerivation rec { machine parts but pretty sure is not what you are looking for when you are more interested in creating computer-animated movies. ''; - homepage = "http://openscad.org/"; + homepage = "https://openscad.org/"; license = lib.licenses.gpl2; platforms = lib.platforms.unix; maintainers = with lib.maintainers; [ bjornfor raskin gebner ]; diff --git a/pkgs/applications/graphics/pikopixel/default.nix b/pkgs/applications/graphics/pikopixel/default.nix index 58a52aee494..0463958b4c6 100644 --- a/pkgs/applications/graphics/pikopixel/default.nix +++ b/pkgs/applications/graphics/pikopixel/default.nix @@ -13,7 +13,7 @@ in gnustep'.gsmakeDerivation rec { version = "1.0-b10"; src = fetchurl { - url = "http://twilightedge.com/downloads/PikoPixel.Sources.${version}.tar.gz"; + url = "https://twilightedge.com/downloads/PikoPixel.Sources.${version}.tar.gz"; sha256 = "1b27npgsan2nx1p581b9q2krx4506yyd6s34r4sf1r9x9adshm77"; }; @@ -38,8 +38,8 @@ in gnustep'.gsmakeDerivation rec { meta = with lib; { description = "Application for drawing and editing pixel-art images"; - homepage = "http://twilightedge.com/mac/pikopixel/"; - downloadPage = "http://twilightedge.com/mac/pikopixel/"; + homepage = "https://twilightedge.com/mac/pikopixel/"; + downloadPage = "https://twilightedge.com/mac/pikopixel/"; license = licenses.agpl3; maintainers = with maintainers; [ fgaz ]; platforms = platforms.all; diff --git a/pkgs/applications/graphics/viewnior/default.nix b/pkgs/applications/graphics/viewnior/default.nix index 74cc2bdc4e8..7f90c815837 100644 --- a/pkgs/applications/graphics/viewnior/default.nix +++ b/pkgs/applications/graphics/viewnior/default.nix @@ -55,7 +55,7 @@ stdenv.mkDerivation rec { in mind (follows Gnome HIG2). ''; license = licenses.gpl3; - homepage = "http://siyanpanayotov.com/project/viewnior/"; + homepage = "https://siyanpanayotov.com/project/viewnior/"; maintainers = with maintainers; [ smironov artturin ]; platforms = platforms.gnu ++ platforms.linux; }; diff --git a/pkgs/applications/misc/yate/default.nix b/pkgs/applications/misc/yate/default.nix index 07857fbf5e9..76eb27a624c 100644 --- a/pkgs/applications/misc/yate/default.nix +++ b/pkgs/applications/misc/yate/default.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation rec { meta = { description = "Yet another telephony engine"; - homepage = "http://yate.ro/"; + homepage = "https://yate.ro/"; # Yate's license is GPL with an exception for linking with # OpenH323 and PWlib (licensed under MPL). license = lib.licenses.gpl2Only; diff --git a/pkgs/applications/networking/irc/kirc/default.nix b/pkgs/applications/networking/irc/kirc/default.nix index b0e771a80f4..c21a444b61c 100644 --- a/pkgs/applications/networking/irc/kirc/default.nix +++ b/pkgs/applications/networking/irc/kirc/default.nix @@ -19,7 +19,7 @@ stdenv.mkDerivation rec { installFlags = [ "PREFIX=$(out)" ]; meta = with lib; { - homepage = "http://kirc.io/"; + homepage = "https://mcpcpc.github.io/kirc/"; description = "Tiny IRC client written in C99"; longDescription = '' kirc is a tiny open-source Internet Relay Chat (IRC) client designed with diff --git a/pkgs/applications/networking/irc/kvirc/default.nix b/pkgs/applications/networking/irc/kvirc/default.nix index 8c1a96d5234..eb443666764 100644 --- a/pkgs/applications/networking/irc/kvirc/default.nix +++ b/pkgs/applications/networking/irc/kvirc/default.nix @@ -24,7 +24,7 @@ mkDerivation rec { meta = with lib; { description = "Advanced IRC Client"; - homepage = "http://www.kvirc.net/"; + homepage = "https://www.kvirc.net/"; license = licenses.gpl2; maintainers = [ maintainers.suhr ]; platforms = platforms.linux; diff --git a/pkgs/applications/networking/p2p/mldonkey/default.nix b/pkgs/applications/networking/p2p/mldonkey/default.nix index 7bc5e2f5967..81a625e5210 100644 --- a/pkgs/applications/networking/p2p/mldonkey/default.nix +++ b/pkgs/applications/networking/p2p/mldonkey/default.nix @@ -39,7 +39,7 @@ stdenv.mkDerivation rec { meta = { broken = stdenv.isDarwin; description = "Client for many p2p networks, with multiple frontends"; - homepage = "http://mldonkey.sourceforge.net/"; + homepage = "https://github.com/ygrek/mldonkey"; license = lib.licenses.gpl2Only; platforms = lib.platforms.unix; }; diff --git a/pkgs/applications/radio/ebook2cw/default.nix b/pkgs/applications/radio/ebook2cw/default.nix index 6c3fbc16c40..329d87d80f4 100644 --- a/pkgs/applications/radio/ebook2cw/default.nix +++ b/pkgs/applications/radio/ebook2cw/default.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Convert ebooks to Morse MP3s/OGGs"; - homepage = "http://fkurz.net/ham/ebook2cw.html"; + homepage = "https://fkurz.net/ham/ebook2cw.html"; license = licenses.gpl2; platforms = platforms.all; maintainers = with maintainers; [ earldouglas ]; diff --git a/pkgs/applications/science/logic/aiger/default.nix b/pkgs/applications/science/logic/aiger/default.nix index 15c45466b13..4f3f585c220 100644 --- a/pkgs/applications/science/logic/aiger/default.nix +++ b/pkgs/applications/science/logic/aiger/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation rec { version = "1.9.9"; src = fetchurl { - url = "http://fmv.jku.at/aiger/${pname}-${version}.tar.gz"; + url = "https://fmv.jku.at/aiger/${pname}-${version}.tar.gz"; sha256 = "1ish0dw0nf9gyghxsdhpy1jjiy5wp54c993swp85xp7m6vdx6l0y"; }; @@ -47,7 +47,7 @@ stdenv.mkDerivation rec { meta = { description = "And-Inverter Graph (AIG) utilities"; - homepage = "http://fmv.jku.at/aiger/"; + homepage = "https://fmv.jku.at/aiger/"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ thoughtpolice ]; platforms = lib.platforms.unix; diff --git a/pkgs/applications/science/logic/cadical/default.nix b/pkgs/applications/science/logic/cadical/default.nix index 9bfadc41f60..a9b27877ab1 100644 --- a/pkgs/applications/science/logic/cadical/default.nix +++ b/pkgs/applications/science/logic/cadical/default.nix @@ -43,6 +43,6 @@ stdenv.mkDerivation rec { maintainers = with maintainers; [ shnarazk ]; platforms = platforms.unix; license = licenses.mit; - homepage = "http://fmv.jku.at/cadical"; + homepage = "https://fmv.jku.at/cadical/"; }; } diff --git a/pkgs/applications/science/logic/cvc3/default.nix b/pkgs/applications/science/logic/cvc3/default.nix index cfa8f62990c..0385909610e 100644 --- a/pkgs/applications/science/logic/cvc3/default.nix +++ b/pkgs/applications/science/logic/cvc3/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation rec { version = "2.4.1"; src = fetchurl { - url = "http://www.cs.nyu.edu/acsys/cvc3/releases/${version}/${pname}-${version}.tar.gz"; + url = "https://cs.nyu.edu/acsys/cvc3/releases/${version}/${pname}-${version}.tar.gz"; sha256 = "1xxcwhz3y6djrycw8sm6xz83wb4hb12rd1n0skvc7fng0rh1snym"; }; @@ -32,11 +32,11 @@ stdenv.mkDerivation rec { [ raskin ]; platforms = platforms.unix; license = licenses.free; - homepage = "http://www.cs.nyu.edu/acsys/cvc3/index.html"; + homepage = "https://cs.nyu.edu/acsys/cvc3/index.html"; }; passthru = { updateInfo = { - downloadPage = "http://www.cs.nyu.edu/acsys/cvc3/download.html"; + downloadPage = "https://cs.nyu.edu/acsys/cvc3/download.html"; }; }; } diff --git a/pkgs/applications/science/logic/kissat/default.nix b/pkgs/applications/science/logic/kissat/default.nix index 2941952287b..d1703340527 100644 --- a/pkgs/applications/science/logic/kissat/default.nix +++ b/pkgs/applications/science/logic/kissat/default.nix @@ -48,6 +48,6 @@ stdenv.mkDerivation rec { maintainers = with maintainers; [ shnarazk ]; platforms = platforms.unix; license = licenses.mit; - homepage = "http://fmv.jku.at/kissat"; + homepage = "https://fmv.jku.at/kissat"; }; } diff --git a/pkgs/applications/science/logic/picosat/default.nix b/pkgs/applications/science/logic/picosat/default.nix index 48def5fc2e4..1fef05069a6 100644 --- a/pkgs/applications/science/logic/picosat/default.nix +++ b/pkgs/applications/science/logic/picosat/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation rec { version = "965"; src = fetchurl { - url = "http://fmv.jku.at/picosat/${pname}-${version}.tar.gz"; + url = "https://fmv.jku.at/picosat/${pname}-${version}.tar.gz"; sha256 = "0m578rpa5rdn08d10kr4lbsdwp4402hpavrz6n7n53xs517rn5hm"; }; @@ -36,7 +36,7 @@ stdenv.mkDerivation rec { meta = { description = "SAT solver with proof and core support"; - homepage = "http://fmv.jku.at/picosat/"; + homepage = "https://fmv.jku.at/picosat/"; license = lib.licenses.mit; platforms = lib.platforms.unix; maintainers = with lib.maintainers; [ roconnor thoughtpolice ]; diff --git a/pkgs/applications/science/math/colpack/default.nix b/pkgs/applications/science/math/colpack/default.nix index 3cc9290a762..d5ab38ff751 100644 --- a/pkgs/applications/science/math/colpack/default.nix +++ b/pkgs/applications/science/math/colpack/default.nix @@ -35,7 +35,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A package comprising of implementations of algorithms for vertex coloring and derivative computation"; - homepage = "http://cscapes.cs.purdue.edu/coloringpage/software.htm#functionalities"; + homepage = "https://cscapes.cs.purdue.edu/coloringpage/software.htm#functionalities"; license = licenses.lgpl3Plus; platforms = platforms.unix; maintainers = with maintainers; [ edwtjo ]; diff --git a/pkgs/applications/virtualization/qemu/default.nix b/pkgs/applications/virtualization/qemu/default.nix index ea96f7f241b..0dc91af767f 100644 --- a/pkgs/applications/virtualization/qemu/default.nix +++ b/pkgs/applications/virtualization/qemu/default.nix @@ -266,11 +266,8 @@ stdenv.mkDerivation (finalAttrs: { requiredSystemFeatures = [ "big-parallel" ]; meta = with lib; { - homepage = "http://www.qemu.org/"; - description = - if toolsOnly - then "Support tools for qemu, a machine emulator and virtualizer" - else "A generic and open source machine emulator and virtualizer"; + homepage = "https://www.qemu.org/"; + description = "A generic and open source machine emulator and virtualizer"; license = licenses.gpl2Plus; maintainers = with maintainers; [ eelco qyliss ]; platforms = platforms.unix; diff --git a/pkgs/applications/virtualization/virt-manager/default.nix b/pkgs/applications/virtualization/virt-manager/default.nix index 163812bdccf..4e4d7e7d0a9 100644 --- a/pkgs/applications/virtualization/virt-manager/default.nix +++ b/pkgs/applications/virtualization/virt-manager/default.nix @@ -83,7 +83,7 @@ python3.pkgs.buildPythonApplication rec { ''; meta = with lib; { - homepage = "http://virt-manager.org"; + homepage = "https://virt-manager.org"; description = "Desktop user interface for managing virtual machines"; longDescription = '' The virt-manager application is a desktop user interface for managing diff --git a/pkgs/applications/virtualization/virt-viewer/default.nix b/pkgs/applications/virtualization/virt-viewer/default.nix index 0e24dd9b08f..58d9edb269a 100644 --- a/pkgs/applications/virtualization/virt-viewer/default.nix +++ b/pkgs/applications/virtualization/virt-viewer/default.nix @@ -104,7 +104,7 @@ stdenv.mkDerivation rec { }; passthru = { updateInfo = { - downloadPage = "http://virt-manager.org/download.html"; + downloadPage = "https://virt-manager.org/download.html"; }; }; } diff --git a/pkgs/data/soundfonts/generaluser/default.nix b/pkgs/data/soundfonts/generaluser/default.nix index 5191199e647..abe1ea4ceac 100644 --- a/pkgs/data/soundfonts/generaluser/default.nix +++ b/pkgs/data/soundfonts/generaluser/default.nix @@ -6,7 +6,7 @@ stdenv.mkDerivation rec { # we can't use fetchurl since stdenv does not handle unpacking *.zip's by default. src = fetchzip { - # Linked on http://www.schristiancollins.com/generaluser.php: + # Linked on https://www.schristiancollins.com/generaluser.php: url = "https://www.dropbox.com/s/4x27l49kxcwamp5/GeneralUser_GS_${version}.zip"; sha256 = "sha256-lwUlWubXiVZ8fijKuNF54YQjT0uigjNAbjKaNjmC51s="; }; @@ -17,7 +17,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "a SoundFont bank featuring 259 instrument presets and 11 drum kits"; - homepage = "http://www.schristiancollins.com/generaluser.php"; + homepage = "https://www.schristiancollins.com/generaluser.php"; license = licenses.generaluser; platforms = platforms.all; maintainers = with maintainers; [ ckie ]; diff --git a/pkgs/development/compilers/pforth/default.nix b/pkgs/development/compilers/pforth/default.nix index d0766fe1376..aadc72c174e 100644 --- a/pkgs/development/compilers/pforth/default.nix +++ b/pkgs/development/compilers/pforth/default.nix @@ -38,7 +38,7 @@ stdenv.mkDerivation (finalAttrs: { ''; meta = { - homepage = "http://www.softsynth.com/pforth/"; + homepage = "https://www.softsynth.com/pforth/"; description = "Portable Portable ANS-like Forth written in ANSI 'C'"; changelog = "https://github.com/philburk/pforth/blob/v${finalAttrs.version}/RELEASES.md"; license = lib.licenses.bsd0; diff --git a/pkgs/development/compilers/uasm/default.nix b/pkgs/development/compilers/uasm/default.nix index cae9a17ccca..9fe0e42cb13 100644 --- a/pkgs/development/compilers/uasm/default.nix +++ b/pkgs/development/compilers/uasm/default.nix @@ -54,7 +54,7 @@ stdenv.mkDerivation rec { }; meta = with lib; { - homepage = "http://www.terraspace.co.uk/uasm.html"; + homepage = "https://www.terraspace.co.uk/uasm.html"; description = "A free MASM-compatible assembler based on JWasm"; platforms = platforms.unix; maintainers = with maintainers; [ thiagokokada ]; diff --git a/pkgs/development/interpreters/picolisp/default.nix b/pkgs/development/interpreters/picolisp/default.nix index b85f99a7b52..4e589cb722e 100644 --- a/pkgs/development/interpreters/picolisp/default.nix +++ b/pkgs/development/interpreters/picolisp/default.nix @@ -60,7 +60,7 @@ stdenv.mkDerivation rec { passthru = { updateInfo = { - downloadPage = "http://www.software-lab.de/down.html"; + downloadPage = "https://www.software-lab.de/down.html"; }; }; } diff --git a/pkgs/development/libraries/ctpp2/default.nix b/pkgs/development/libraries/ctpp2/default.nix index c8bc708a2aa..5aac0850a74 100644 --- a/pkgs/development/libraries/ctpp2/default.nix +++ b/pkgs/development/libraries/ctpp2/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation rec { version = "2.8.3"; src = fetchurl { - url = "http://ctpp.havoc.ru/download/${pname}-${version}.tar.gz"; + url = "https://ctpp.havoc.ru/download/${pname}-${version}.tar.gz"; sha256 = "1z22zfw9lb86z4hcan9hlvji49c9b7vznh7gjm95gnvsh43zsgx8"; }; @@ -25,7 +25,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A high performance templating engine"; - homepage = "http://ctpp.havoc.ru"; + homepage = "https://ctpp.havoc.ru/"; maintainers = [ maintainers.robbinch ]; platforms = platforms.linux; license = licenses.bsd2; diff --git a/pkgs/development/libraries/dbus/default.nix b/pkgs/development/libraries/dbus/default.nix index e9f0f013fe2..a809eda31c0 100644 --- a/pkgs/development/libraries/dbus/default.nix +++ b/pkgs/development/libraries/dbus/default.nix @@ -114,7 +114,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Simple interprocess messaging system"; - homepage = "http://www.freedesktop.org/wiki/Software/dbus/"; + homepage = "https://www.freedesktop.org/wiki/Software/dbus/"; changelog = "https://gitlab.freedesktop.org/dbus/dbus/-/blob/dbus-${version}/NEWS"; license = licenses.gpl2Plus; # most is also under AFL-2.1 maintainers = teams.freedesktop.members ++ (with maintainers; [ ]); diff --git a/pkgs/development/libraries/irrlicht/default.nix b/pkgs/development/libraries/irrlicht/default.nix index 7a68f130663..fa5d2b3b0c9 100644 --- a/pkgs/development/libraries/irrlicht/default.nix +++ b/pkgs/development/libraries/irrlicht/default.nix @@ -37,7 +37,7 @@ stdenv.mkDerivation rec { ] ++ lib.optional stdenv.isAarch64 zlib; meta = { - homepage = "http://irrlicht.sourceforge.net/"; + homepage = "https://irrlicht.sourceforge.io/"; license = lib.licenses.zlib; description = "Open source high performance realtime 3D engine written in C++"; platforms = lib.platforms.linux ++ lib.platforms.darwin; diff --git a/pkgs/development/libraries/irrlicht/mac.nix b/pkgs/development/libraries/irrlicht/mac.nix index a7901466081..b3ef27e3877 100644 --- a/pkgs/development/libraries/irrlicht/mac.nix +++ b/pkgs/development/libraries/irrlicht/mac.nix @@ -35,7 +35,7 @@ stdenv.mkDerivation rec { buildInputs = [ OpenGL Cocoa IOKit ]; meta = { - homepage = "http://irrlicht.sourceforge.net/"; + homepage = "https://irrlicht.sourceforge.net/"; license = lib.licenses.zlib; description = "Open source high performance realtime 3D engine written in C++"; platforms = lib.platforms.darwin; diff --git a/pkgs/development/libraries/libcdr/default.nix b/pkgs/development/libraries/libcdr/default.nix index b3087f0abea..64695aaa55d 100644 --- a/pkgs/development/libraries/libcdr/default.nix +++ b/pkgs/development/libraries/libcdr/default.nix @@ -19,7 +19,7 @@ stdenv.mkDerivation rec { meta = { description = "A library providing ability to interpret and import Corel Draw drawings into various applications"; - homepage = "http://www.freedesktop.org/wiki/Software/libcdr"; + homepage = "https://wiki.documentfoundation.org/DLP/Libraries/libcdr"; platforms = lib.platforms.all; license = lib.licenses.mpl20; }; diff --git a/pkgs/development/libraries/libevdev/default.nix b/pkgs/development/libraries/libevdev/default.nix index 90853d5f2db..c8db600dce8 100644 --- a/pkgs/development/libraries/libevdev/default.nix +++ b/pkgs/development/libraries/libevdev/default.nix @@ -13,7 +13,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Wrapper library for evdev devices"; - homepage = "http://www.freedesktop.org/software/libevdev/doc/latest/index.html"; + homepage = "https://www.freedesktop.org/software/libevdev/doc/latest/index.html"; license = licenses.mit; platforms = platforms.linux; maintainers = [ maintainers.amorsillo ]; diff --git a/pkgs/development/libraries/libproxy/default.nix b/pkgs/development/libraries/libproxy/default.nix index 62ddcd67390..5c7a3162502 100644 --- a/pkgs/development/libraries/libproxy/default.nix +++ b/pkgs/development/libraries/libproxy/default.nix @@ -76,7 +76,7 @@ stdenv.mkDerivation rec { meta = with lib; { platforms = platforms.linux ++ platforms.darwin; license = licenses.lgpl21; - homepage = "http://libproxy.github.io/libproxy/"; + homepage = "https://libproxy.github.io/libproxy/"; description = "A library that provides automatic proxy configuration management"; }; } diff --git a/pkgs/development/libraries/libsodium/default.nix b/pkgs/development/libraries/libsodium/default.nix index 14e730d69e5..b4eeed32d59 100644 --- a/pkgs/development/libraries/libsodium/default.nix +++ b/pkgs/development/libraries/libsodium/default.nix @@ -32,7 +32,7 @@ stdenv.mkDerivation (finalAttrs: { meta = with lib; { description = "A modern and easy-to-use crypto library"; - homepage = "http://doc.libsodium.org/"; + homepage = "https://doc.libsodium.org/"; license = licenses.isc; maintainers = with maintainers; [ raskin ]; pkgConfigModules = [ "libsodium" ]; diff --git a/pkgs/development/libraries/libtsm/default.nix b/pkgs/development/libraries/libtsm/default.nix index 75773296fab..c8e00cf4a94 100644 --- a/pkgs/development/libraries/libtsm/default.nix +++ b/pkgs/development/libraries/libtsm/default.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Terminal-emulator State Machine"; - homepage = "http://www.freedesktop.org/wiki/Software/kmscon/libtsm/"; + homepage = "https://www.freedesktop.org/wiki/Software/kmscon/libtsm/"; license = licenses.mit; maintainers = with maintainers; [ ]; platforms = platforms.linux; diff --git a/pkgs/development/libraries/openfx/default.nix b/pkgs/development/libraries/openfx/default.nix index 201bf16ad64..2bfb0870858 100644 --- a/pkgs/development/libraries/openfx/default.nix +++ b/pkgs/development/libraries/openfx/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation { version = "1.4"; src = fetchFromGitHub { - owner = "ofxa"; + owner = "AcademySoftwareFoundation"; repo = "openfx"; rev = "OFX_Release_1_4_TAG"; sha256 = "0k9ggzr6bisn77mipjfvawg3mv4bz50b63v8f7w1jhldi1sfy548"; @@ -27,7 +27,7 @@ stdenv.mkDerivation { meta = with lib; { description = "Image processing plug-in standard"; - homepage = "http://openeffects.org/"; + homepage = "https://openeffects.org/"; license = licenses.bsd3; platforms = platforms.all; maintainers = [ maintainers.guibou ]; diff --git a/pkgs/development/libraries/polkit/default.nix b/pkgs/development/libraries/polkit/default.nix index 5dbb7dc98b2..80badcdd4db 100644 --- a/pkgs/development/libraries/polkit/default.nix +++ b/pkgs/development/libraries/polkit/default.nix @@ -181,7 +181,7 @@ stdenv.mkDerivation rec { ''; meta = with lib; { - homepage = "http://www.freedesktop.org/wiki/Software/polkit"; + homepage = "https://gitlab.freedesktop.org/polkit/polkit/"; description = "A toolkit for defining and handling the policy that allows unprivileged processes to speak to privileged processes"; license = licenses.lgpl2Plus; platforms = platforms.linux; diff --git a/pkgs/development/libraries/portaudio/default.nix b/pkgs/development/libraries/portaudio/default.nix index 589cf7cdf95..a109aad97b9 100644 --- a/pkgs/development/libraries/portaudio/default.nix +++ b/pkgs/development/libraries/portaudio/default.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { version = "190700_20210406"; src = fetchurl { - url = "http://files.portaudio.com/archives/pa_stable_v${version}.tgz"; + url = "https://files.portaudio.com/archives/pa_stable_v${version}.tgz"; sha256 = "1vrdrd42jsnffh6rq8ap2c6fr4g9fcld89z649fs06bwqx1bzvs7"; }; @@ -54,7 +54,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Portable cross-platform Audio API"; - homepage = "http://www.portaudio.com/"; + homepage = "https://www.portaudio.com/"; # Not exactly a bsd license, but alike license = licenses.mit; maintainers = with maintainers; [ lovek323 ]; diff --git a/pkgs/development/libraries/startup-notification/default.nix b/pkgs/development/libraries/startup-notification/default.nix index 001123ead60..c1c5819ad42 100644 --- a/pkgs/development/libraries/startup-notification/default.nix +++ b/pkgs/development/libraries/startup-notification/default.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { buildInputs = [ libX11 libxcb xcbutil ]; meta = { - homepage = "http://www.freedesktop.org/software/startup-notification"; + homepage = "https://www.freedesktop.org/software/startup-notification"; description = "Application startup notification and feedback library"; license = lib.licenses.lgpl2; }; diff --git a/pkgs/development/libraries/vcg/default.nix b/pkgs/development/libraries/vcg/default.nix index 8b86df2e1c1..e7e818cbea5 100644 --- a/pkgs/development/libraries/vcg/default.nix +++ b/pkgs/development/libraries/vcg/default.nix @@ -20,7 +20,7 @@ stdenv.mkDerivation rec { ''; meta = with lib; { - homepage = "http://vcg.isti.cnr.it/vcglib/install.html"; + homepage = "https://vcg.isti.cnr.it/vcglib/install.html"; description = "C++ library for manipulation, processing and displaying with OpenGL of triangle and tetrahedral meshes"; license = licenses.gpl3; platforms = platforms.all; diff --git a/pkgs/development/lisp-modules-new-obsolete/examples/bordeaux-threads.nix b/pkgs/development/lisp-modules-new-obsolete/examples/bordeaux-threads.nix index 42afb6d2706..31a53b0f949 100644 --- a/pkgs/development/lisp-modules-new-obsolete/examples/bordeaux-threads.nix +++ b/pkgs/development/lisp-modules-new-obsolete/examples/bordeaux-threads.nix @@ -33,7 +33,7 @@ let pname = "bordeaux-threads"; version = "0.8.8"; src = pkgs.fetchzip { - url = "http://github.com/sionescu/bordeaux-threads/archive/v0.8.8.tar.gz"; + url = "https://github.com/sionescu/bordeaux-threads/archive/v0.8.8.tar.gz"; sha256 = "19i443fz3488v1pbbr9x24y8h8vlyhny9vj6c9jk5prm702awrp6"; }; lisp = sbcl; diff --git a/pkgs/development/python-modules/cvxopt/default.nix b/pkgs/development/python-modules/cvxopt/default.nix index 0a2b13ad4c8..f6135ecba74 100644 --- a/pkgs/development/python-modules/cvxopt/default.nix +++ b/pkgs/development/python-modules/cvxopt/default.nix @@ -68,7 +68,7 @@ buildPythonPackage rec { unittestFlagsArray = [ "-s" "tests" ]; meta = with lib; { - homepage = "http://cvxopt.org/"; + homepage = "https://cvxopt.org/"; description = "Python Software for Convex Optimization"; longDescription = '' CVXOPT is a free software package for convex optimization based on the diff --git a/pkgs/development/python-modules/polling/default.nix b/pkgs/development/python-modules/polling/default.nix index 22e7e0c3916..ee816585893 100644 --- a/pkgs/development/python-modules/polling/default.nix +++ b/pkgs/development/python-modules/polling/default.nix @@ -35,7 +35,7 @@ buildPythonPackage rec { meta = with lib; { description = "Powerful polling utility in Python"; - homepage = "http://github.com/justiniso/polling"; + homepage = "https://github.com/justiniso/polling"; license = licenses.mit; maintainers = with maintainers; [ ]; }; diff --git a/pkgs/development/python-modules/sqlobject/default.nix b/pkgs/development/python-modules/sqlobject/default.nix index 6a4ea78b82e..06d2534eb23 100644 --- a/pkgs/development/python-modules/sqlobject/default.nix +++ b/pkgs/development/python-modules/sqlobject/default.nix @@ -44,7 +44,7 @@ buildPythonPackage rec { meta = with lib; { description = "Object Relational Manager for providing an object interface to your database"; - homepage = "http://www.sqlobject.org/"; + homepage = "https://www.sqlobject.org/"; changelog = "https://github.com/sqlobject/sqlobject/blob/${version}/docs/News.rst"; license = licenses.lgpl21Only; maintainers = with maintainers; [ ]; diff --git a/pkgs/development/tools/nailgun/default.nix b/pkgs/development/tools/nailgun/default.nix index 2d8b075a685..6c71e374ffc 100644 --- a/pkgs/development/tools/nailgun/default.nix +++ b/pkgs/development/tools/nailgun/default.nix @@ -11,7 +11,7 @@ let commonMeta = { license = lib.licenses.asl20; - homepage = "http://www.martiansoftware.com/nailgun/"; + homepage = "https://www.martiansoftware.com/nailgun/"; platforms = lib.platforms.linux; maintainers = with lib.maintainers; [ ]; }; diff --git a/pkgs/games/hedgewars/default.nix b/pkgs/games/hedgewars/default.nix index 6674c85c332..94f8a28add9 100644 --- a/pkgs/games/hedgewars/default.nix +++ b/pkgs/games/hedgewars/default.nix @@ -56,7 +56,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Turn-based strategy artillery game similar to Worms"; - homepage = "http://hedgewars.org/"; + homepage = "https://hedgewars.org/"; license = licenses.gpl2; longDescription = '' Each player controls a team of several hedgehogs. During the course of diff --git a/pkgs/games/hyperrogue/default.nix b/pkgs/games/hyperrogue/default.nix index cd24a44f92a..2735cfff194 100644 --- a/pkgs/games/hyperrogue/default.nix +++ b/pkgs/games/hyperrogue/default.nix @@ -60,7 +60,7 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; meta = with lib; { - homepage = "http://www.roguetemple.com/z/hyper/"; + homepage = "https://www.roguetemple.com/z/hyper/"; description = "A roguelike game set in hyperbolic geometry"; maintainers = with maintainers; [ rardiol ]; license = licenses.gpl2; diff --git a/pkgs/games/xskat/default.nix b/pkgs/games/xskat/default.nix index 74c2be586ce..6baf397ad67 100644 --- a/pkgs/games/xskat/default.nix +++ b/pkgs/games/xskat/default.nix @@ -8,7 +8,7 @@ stdenv.mkDerivation rec { buildInputs = [ libX11 imake ]; src = fetchurl { - url = "http://www.xskat.de/xskat-${version }.tar.gz"; + url = "https://web.archive.org/web/20220331112433if_/https://www.xskat.de/xskat-${version}.tar.gz"; sha256 = "8ba52797ccbd131dce69b96288f525b0d55dee5de4008733f7a5a51deb831c10"; }; @@ -25,6 +25,6 @@ stdenv.mkDerivation rec { platforms = platforms.unix; license = licenses.free; longDescription = "Play the german card game Skat against the AI or over IRC."; - homepage = "http://www.xskat.de/"; + homepage = "https://web.archive.org/web/20221003060115/https://www.xskat.de/xskat.html"; }; } diff --git a/pkgs/misc/cups/cups-pk-helper.nix b/pkgs/misc/cups/cups-pk-helper.nix index f5edc729340..1a3b4ebfbef 100644 --- a/pkgs/misc/cups/cups-pk-helper.nix +++ b/pkgs/misc/cups/cups-pk-helper.nix @@ -22,7 +22,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "PolicyKit helper to configure cups with fine-grained privileges"; - homepage = "http://www.freedesktop.org/wiki/Software/cups-pk-helper/"; + homepage = "https://www.freedesktop.org/wiki/Software/cups-pk-helper/"; license = licenses.gpl2; platforms = platforms.linux; maintainers = [ maintainers.bjornfor ]; diff --git a/pkgs/os-specific/linux/ch9344/default.nix b/pkgs/os-specific/linux/ch9344/default.nix index 8f258d227bb..e7da864b90c 100644 --- a/pkgs/os-specific/linux/ch9344/default.nix +++ b/pkgs/os-specific/linux/ch9344/default.nix @@ -37,7 +37,7 @@ stdenv.mkDerivation rec { ''; meta = with lib; { - homepage = "http://www.wch-ic.com/"; + homepage = "https://www.wch-ic.com/"; downloadPage = "https://www.wch.cn/downloads/CH9344SER_LINUX_ZIP.html"; description = "WCH CH9344/CH348 UART driver"; longDescription = '' diff --git a/pkgs/os-specific/linux/kmscon/default.nix b/pkgs/os-specific/linux/kmscon/default.nix index 0d146613542..4762b63eda9 100644 --- a/pkgs/os-specific/linux/kmscon/default.nix +++ b/pkgs/os-specific/linux/kmscon/default.nix @@ -72,7 +72,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "KMS/DRM based System Console"; - homepage = "http://www.freedesktop.org/wiki/Software/kmscon/"; + homepage = "https://www.freedesktop.org/wiki/Software/kmscon/"; license = licenses.mit; maintainers = with maintainers; [ omasanori ]; platforms = platforms.linux; diff --git a/pkgs/os-specific/linux/microcode/intel.nix b/pkgs/os-specific/linux/microcode/intel.nix index b58d471680b..c489e746886 100644 --- a/pkgs/os-specific/linux/microcode/intel.nix +++ b/pkgs/os-specific/linux/microcode/intel.nix @@ -25,7 +25,7 @@ stdenv.mkDerivation rec { ''; meta = with lib; { - homepage = "http://www.intel.com/"; + homepage = "https://www.intel.com/"; description = "Microcode for Intel processors"; license = licenses.unfreeRedistributableFirmware; platforms = platforms.linux; diff --git a/pkgs/tools/filesystems/unionfs-fuse/default.nix b/pkgs/tools/filesystems/unionfs-fuse/default.nix index 8f008c3a346..4dc9ca9de6e 100644 --- a/pkgs/tools/filesystems/unionfs-fuse/default.nix +++ b/pkgs/tools/filesystems/unionfs-fuse/default.nix @@ -14,7 +14,7 @@ stdenv.mkDerivation rec { patches = [ # Prevent the unionfs daemon from being killed during # shutdown. See - # http://www.freedesktop.org/wiki/Software/systemd/RootStorageDaemons/ + # https://www.freedesktop.org/wiki/Software/systemd/RootStorageDaemons/ # for details. ./prevent-kill-on-shutdown.patch ]; diff --git a/pkgs/tools/misc/ccal/default.nix b/pkgs/tools/misc/ccal/default.nix index 55ccb7f774c..e7a01795f0d 100644 --- a/pkgs/tools/misc/ccal/default.nix +++ b/pkgs/tools/misc/ccal/default.nix @@ -9,7 +9,7 @@ stdenv.mkDerivation rec { pname = "ccal"; version = "2.5.3"; src = fetchurl { - url = "http://ccal.chinesebay.com/ccal/${pname}-${version}.tar.gz"; + url = "https://ccal.chinesebay.com/${pname}-${version}.tar.gz"; sha256 = "sha256-PUy9yfkFzgKrSEBB+79/C3oxmuajUMbBbWNuGlpQ35Y="; }; @@ -25,11 +25,10 @@ stdenv.mkDerivation rec { ''; meta = { - homepage = "http://ccal.chinesebay.com/ccal.htm"; + homepage = "https://ccal.chinesebay.com/ccal.htm"; description = "Command line Chinese calendar viewer, similar to cal"; license = lib.licenses.gpl3; maintainers = with lib.maintainers; [ sharzy ]; platforms = lib.platforms.all; }; } - diff --git a/pkgs/tools/misc/desktop-file-utils/default.nix b/pkgs/tools/misc/desktop-file-utils/default.nix index 218eed5eb8b..b58a7d819ba 100644 --- a/pkgs/tools/misc/desktop-file-utils/default.nix +++ b/pkgs/tools/misc/desktop-file-utils/default.nix @@ -50,7 +50,7 @@ stdenv.mkDerivation rec { setupHook = ./setup-hook.sh; meta = with lib; { - homepage = "http://www.freedesktop.org/wiki/Software/desktop-file-utils"; + homepage = "https://www.freedesktop.org/wiki/Software/desktop-file-utils"; description = "Command line utilities for working with .desktop files"; platforms = platforms.linux ++ platforms.darwin; license = licenses.gpl2Plus; diff --git a/pkgs/tools/misc/plantuml/default.nix b/pkgs/tools/misc/plantuml/default.nix index 9a6698ecfd4..eb6cfb7ebcb 100644 --- a/pkgs/tools/misc/plantuml/default.nix +++ b/pkgs/tools/misc/plantuml/default.nix @@ -25,7 +25,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Draw UML diagrams using a simple and human readable text description"; - homepage = "http://plantuml.sourceforge.net/"; + homepage = "https://plantuml.com/"; sourceProvenance = with sourceTypes; [ binaryBytecode ]; # "plantuml -license" says GPLv3 or later license = licenses.gpl3Plus; diff --git a/pkgs/tools/networking/redsocks/default.nix b/pkgs/tools/networking/redsocks/default.nix index 9b8989f30c2..cebec2cb2b6 100644 --- a/pkgs/tools/networking/redsocks/default.nix +++ b/pkgs/tools/networking/redsocks/default.nix @@ -22,7 +22,7 @@ stdenv.mkDerivation rec { meta = { description = "Transparent redirector of any TCP connection to proxy"; - homepage = "http://darkk.net.ru/redsocks/"; + homepage = "https://darkk.net.ru/redsocks/"; license = lib.licenses.asl20; maintainers = [ lib.maintainers.ekleog ]; platforms = lib.platforms.linux; diff --git a/pkgs/tools/networking/ripmime/default.nix b/pkgs/tools/networking/ripmime/default.nix index 6584edca5e6..7cfb31ff985 100644 --- a/pkgs/tools/networking/ripmime/default.nix +++ b/pkgs/tools/networking/ripmime/default.nix @@ -4,7 +4,7 @@ stdenv.mkDerivation rec { pname = "ripmime"; version = "1.4.0.10"; src = fetchurl { - url = "http://www.pldaniels.com/${pname}/${pname}-${version}.tar.gz"; + url = "https://pldaniels.com/${pname}/${pname}-${version}.tar.gz"; sha256 = "0sj06ibmlzy34n8v0mnlq2gwidy7n2aqcwgjh0xssz3vi941aqc9"; }; @@ -19,13 +19,13 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Attachment extractor for MIME messages"; maintainers = with maintainers; [ raskin ]; - homepage = "http://www.pldaniels.com/ripmime/"; + homepage = "https://pldaniels.com/ripmime/"; platforms = platforms.all; }; passthru = { updateInfo = { - downloadPage = "http://www.pldaniels.com/ripmime/"; + downloadPage = "https://pldaniels.com/ripmime/"; }; }; } diff --git a/pkgs/tools/security/afl/qemu.nix b/pkgs/tools/security/afl/qemu.nix index 9a8b4b3ca36..845d9fa3e6f 100644 --- a/pkgs/tools/security/afl/qemu.nix +++ b/pkgs/tools/security/afl/qemu.nix @@ -13,7 +13,7 @@ stdenv.mkDerivation rec { srcs = [ (fetchurl { - url = "http://wiki.qemu.org/download/qemu-${version}.tar.bz2"; + url = "https://download.qemu.org/qemu-${version}.tar.bz2"; sha256 = "0j3dfxzrzdp1w21k21fjvmakzc6lcha1rsclaicwqvbf63hkk7vy"; }) afl.src @@ -68,7 +68,7 @@ stdenv.mkDerivation rec { ]; meta = with lib; { - homepage = "http://www.qemu.org/"; + homepage = "https://www.qemu.org/"; description = "Fork of QEMU with AFL instrumentation support"; license = licenses.gpl2Plus; maintainers = with maintainers; [ thoughtpolice ]; diff --git a/pkgs/tools/security/chkrootkit/default.nix b/pkgs/tools/security/chkrootkit/default.nix index b365fa62f95..7c3af075a55 100644 --- a/pkgs/tools/security/chkrootkit/default.nix +++ b/pkgs/tools/security/chkrootkit/default.nix @@ -5,7 +5,7 @@ stdenv.mkDerivation rec { version = "0.55"; src = fetchurl { - url = "ftp://ftp.pangeia.com.br/pub/seg/pac/${pname}-${version}.tar.gz"; + url = "ftp://ftp.chkrootkit.org/pub/seg/pac/${pname}-${version}.tar.gz"; sha256 = "sha256-qBwChuxEkxP5U3ASAqAOgbIE/Cz0PieFhaEcEqXgJYs="; }; @@ -29,7 +29,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Locally checks for signs of a rootkit"; - homepage = "http://www.chkrootkit.org/"; + homepage = "https://www.chkrootkit.org/"; license = licenses.bsd2; platforms = with platforms; linux; }; -- cgit 1.4.1 From 9ec104bb2d35314ffdf2f59c58c80e08c926d27b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 13 Nov 2023 23:57:52 +0100 Subject: lib.fileset: Very minor changes --- lib/fileset/README.md | 3 --- lib/fileset/default.nix | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 8518d88a7d6..91f892a1be9 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -241,7 +241,4 @@ Arguments: ## To update in the future Here's a list of places in the library that need to be updated in the future: -- > The file set library is currently somewhat limited but is being expanded to include more functions over time. - - in [the manual](../../doc/functions/fileset.section.md) - If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 372d445269f..7d5bbeee3ba 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -188,7 +188,7 @@ in { - Set `root` to ${toString fileset._internalBase} or any directory higher up. This changes the layout of the resulting store path. - Set `fileset` to a file set that cannot contain files outside the `root` (${toString root}). This could change the files included in the result.'' else - builtins.seq sourceFilter + seq sourceFilter cleanSourceWith { name = "source"; src = root; -- cgit 1.4.1 From e93a3d7b586022f900f176cf46e7a3bf211cfaf9 Mon Sep 17 00:00:00 2001 From: DS Date: Mon, 23 Oct 2023 19:51:50 -0700 Subject: doc: separate commit header conventions for each area, info on docs changes. --- CONTRIBUTING.md | 33 ++++++++++----------------------- doc/README.md | 21 +++++++++++++++++++++ lib/README.md | 20 ++++++++++++++++++++ nixos/README.md | 21 +++++++++++++++++++++ pkgs/README.md | 19 +++++++++++++++++++ 5 files changed, 91 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06b9c10dfec..ed1b04cf5fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -322,6 +322,8 @@ All the review template samples provided in this section are generic and meant a To get more information about how to review specific parts of Nixpkgs, refer to the documents linked to in the [overview section][overview]. +If a pull request contains documentation changes that might require feedback from the documentation team, ping @NixOS/documentation-team on the pull request. + If you consider having enough knowledge and experience in a topic and would like to be a long-term reviewer for related submissions, please contact the current reviewers for that topic. They will give you information about the reviewing process. The main reviewers for a topic can be hard to find as there is no list, but checking past pull requests to see who reviewed or git-blaming the code to see who committed to that topic can give some hints. Container system, boot system and library changes are some examples of the pull requests fitting this category. @@ -512,34 +514,19 @@ To get a sense for what changes are considered mass rebuilds, see [previously me - If you have commits `pkg-name: oh, forgot to insert whitespace`: squash commits in this case. Use `git rebase -i`. -- Format the commit messages in the following way: - - ``` - (pkg-name | nixos/): (from -> to | init at version | refactor | etc) - - (Motivation for change. Link to release notes. Additional information.) - ``` - - For consistency, there should not be a period at the end of the commit message's summary line (the first line of the commit message). +- For consistency, there should not be a period at the end of the commit message's summary line (the first line of the commit message). - Examples: - - * nginx: init at 2.0.1 - * firefox: 54.0.1 -> 55.0 - - https://www.mozilla.org/en-US/firefox/55.0/releasenotes/ - * nixos/hydra: add bazBaz option - - Dual baz behavior is needed to do foo. - * nixos/nginx: refactor config generation - - The old config generation system used impure shell scripts and could break in specific circumstances (see #1234). - - When adding yourself as maintainer, in the same pull request, make a separate +- When adding yourself as maintainer in the same pull request, make a separate commit with the message `maintainers: add `. Add the commit before those making changes to the package or module. See [Nixpkgs Maintainers](./maintainers/README.md) for details. +- Make sure you read about any commit conventions specific to the area you're touching. See: + - [Commit conventions](./pkgs/README.md#commit-conventions) for changes to `pkgs`. + - [Commit conventions](./lib/README.md#commit-conventions) for changes to `lib`. + - [Commit conventions](./nixos/README.md#commit-conventions) for changes to `nixos`. + - [Commit conventions](./doc/README.md#commit-conventions) for changes to `doc`, the Nixpkgs manual. + ### Writing good commit messages In addition to writing properly formatted commit messages, it's important to include relevant information so other developers can later understand *why* a change was made. While this information usually can be found by digging code, mailing list/Discourse archives, pull request discussions or upstream changes, it may require a lot of work. diff --git a/doc/README.md b/doc/README.md index 3f9aff1a38a..9dee2d30d73 100644 --- a/doc/README.md +++ b/doc/README.md @@ -114,3 +114,24 @@ pear watermelon : green fruit with red flesh ``` + +## Commit conventions + +- Make sure you read about the [commit conventions](../CONTRIBUTING.md#commit-conventions) common to Nixpkgs as a whole. + +- If creating a commit purely for documentation changes, format the commit message in the following way: + + ``` + doc: (documentation summary) + + (Motivation for change, relevant links, additional information.) + ``` + + Examples: + + * doc: update the kernel config documentation to use `nix-shell` + * doc: add information about `nix-update-script` + + Closes #216321. + +- If the commit contains more than just documentation changes, follow the commit message format relevant for the rest of the changes. diff --git a/lib/README.md b/lib/README.md index 627086843db..220940bc212 100644 --- a/lib/README.md +++ b/lib/README.md @@ -74,3 +74,23 @@ path/tests/prop.sh # Run the lib.fileset tests fileset/tests.sh ``` + +## Commit conventions + +- Make sure you read about the [commit conventions](../CONTRIBUTING.md#commit-conventions) common to Nixpkgs as a whole. + +- Format the commit messages in the following way: + + ``` + lib.(section): (init | add additional argument | refactor | etc) + + (Motivation for change. Additional information.) + ``` + + Examples: + + * lib.getExe': check arguments + * lib.fileset: Add an additional argument in the design docs + + Closes #264537 + diff --git a/nixos/README.md b/nixos/README.md index b3cd9d234fa..7f258eb393a 100644 --- a/nixos/README.md +++ b/nixos/README.md @@ -8,6 +8,27 @@ https://nixos.org/nixos and in the manual in doc/manual. You can add new module to your NixOS configuration file (usually it’s `/etc/nixos/configuration.nix`). And do `sudo nixos-rebuild test -I nixpkgs= --fast`. +## Commit conventions + +- Make sure you read about the [commit conventions](../CONTRIBUTING.md#commit-conventions) common to Nixpkgs as a whole. + +- Format the commit messages in the following way: + + ``` + nixos/(module): (init module | add setting | refactor | etc) + + (Motivation for change. Link to release notes. Additional information.) + ``` + + Examples: + + * nixos/hydra: add bazBaz option + + Dual baz behavior is needed to do foo. + * nixos/nginx: refactor config generation + + The old config generation system used impure shell scripts and could break in specific circumstances (see #1234). + ## Reviewing contributions When changing the bootloader installation process, extra care must be taken. Grub installations cannot be rolled back, hence changes may break people’s installations forever. For any non-trivial change to the bootloader please file a PR asking for review, especially from \@edolstra. diff --git a/pkgs/README.md b/pkgs/README.md index 642833d968f..fc33ecea199 100644 --- a/pkgs/README.md +++ b/pkgs/README.md @@ -114,6 +114,25 @@ Now that this is out of the way. To add a package to Nixpkgs: 7. Optionally commit the new package and open a pull request [to nixpkgs](https://github.com/NixOS/nixpkgs/pulls), or use [the Patches category](https://discourse.nixos.org/t/about-the-patches-category/477) on Discourse for sending a patch without a GitHub account. +## Commit conventions + +- Make sure you read about the [commit conventions](../CONTRIBUTING.md#commit-conventions) common to Nixpkgs as a whole. + +- Format the commit messages in the following way: + + ``` + (pkg-name): (from -> to | init at version | refactor | etc) + + (Motivation for change. Link to release notes. Additional information.) + ``` + + Examples: + + * nginx: init at 2.0.1 + * firefox: 54.0.1 -> 55.0 + + https://www.mozilla.org/en-US/firefox/55.0/releasenotes/ + ## Category Hierarchy [categories]: #category-hierarchy -- cgit 1.4.1 From e1d833173839985fdb5dfa3dc3d93bf7db374737 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 14 Nov 2023 06:50:54 +0100 Subject: lib.fileset.fileFilter: Minor cleanups and more tests --- lib/fileset/default.nix | 4 ++-- lib/fileset/internal.nix | 4 ++++ lib/fileset/tests.sh | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index fe7b304ba69..54aace0ea14 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -304,7 +304,7 @@ in { fileFilter (file: hasPrefix "." file.name) ./. # Include all regular files (not symlinks or others) in the current directory - fileFilter (file: file.type == "regular") + fileFilter (file: file.type == "regular") ./. */ fileFilter = /* @@ -325,7 +325,7 @@ in { fileset: if ! isFunction predicate then throw '' - lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function.'' + lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' else _fileFilter predicate (_coerce "lib.fileset.fileFilter: Second argument" fileset); diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index d55c84a395e..a4a230a4798 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -728,8 +728,12 @@ rec { _differenceTree (path + "/${name}") lhsValue (rhs.${name} or null) ) (_directoryEntries path lhs); + # Filters all files in a file set based on a predicate + # Type: ({ name, type, ... } -> Bool) -> FileSet -> FileSet _fileFilter = predicate: fileset: let + # Check the predicate for a path and a filesetTree, returning a new filesetTree + # Type: Path -> filesetTree -> filesetTree recurse = path: tree: mapAttrs (name: subtree: if isAttrs subtree || subtree == "directory" then diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index c1c67800f5e..4247c71efcf 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -785,6 +785,13 @@ checkFileset 'difference ./. ./b' ## File filter +# The first argument needs to be a function +expectFailure 'fileFilter null (abort "this is not needed")' 'lib.fileset.fileFilter: First argument is of type null, but it should be a function instead.' + +# The second argument can be a file set or an existing path +expectFailure 'fileFilter (file: abort "this is not needed") null' 'lib.fileset.fileFilter: Second argument is of type null, but it should be a file set or a path instead.' +expectFailure 'fileFilter (file: abort "this is not needed") ./a' 'lib.fileset.fileFilter: Second argument \('"$work"'/a\) is a path that does not exist.' + # The predicate is not called when there's no files tree=() checkFileset 'fileFilter (file: abort "this is not needed") ./.' -- cgit 1.4.1 From 2035f8a3247a0fce215963e16a161d9387f4b070 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 14 Nov 2023 07:25:15 +0100 Subject: lib.fileset.fileFilter: Don't run predicate unnecessarily Before: nix-repl> fileset.trace (fileset.fileFilter (file: builtins.trace file.name false) ./default.nix) trace: README.md trace: benchmark.sh trace: default.nix trace: internal.nix trace: mock-splitRoot.nix trace: tests.sh After: nix-repl> fileset.trace (fileset.fileFilter (file: builtins.trace file.name false) ./default.nix) trace: default.nix --- lib/fileset/internal.nix | 39 +++++++++++++++++++++++---------------- lib/fileset/tests.sh | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index a4a230a4798..622e9506679 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -732,29 +732,36 @@ rec { # Type: ({ name, type, ... } -> Bool) -> FileSet -> FileSet _fileFilter = predicate: fileset: let - # Check the predicate for a path and a filesetTree, returning a new filesetTree - # Type: Path -> filesetTree -> filesetTree - recurse = path: tree: + # Check the predicate for a single file + # Type: String -> String -> filesetTree + fromFile = name: type: + if + predicate { + inherit name type; + # To ensure forwards compatibility with more arguments being added in the future, + # adding an attribute which can't be deconstructed :) + "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file }:`, use `{ name, file, ... }:` instead." = null; + } + then + type + else + null; + + # Check the predicate for all files in a directory + # Type: Path -> filesetTree + fromDir = path: tree: mapAttrs (name: subtree: if isAttrs subtree || subtree == "directory" then - recurse (path + "/${name}") subtree - else if - predicate { - inherit name; - type = subtree; - # To ensure forwards compatibility with more arguments being added in the future, - # adding an attribute which can't be deconstructed :) - "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file }:`, use `{ name, file, ... }:` instead." = null; - } - then - subtree - else + fromDir (path + "/${name}") subtree + else if subtree == null then null + else + fromFile name subtree ) (_directoryEntries path tree); in if fileset._internalIsEmptyWithoutBase then _emptyWithoutBase else _create fileset._internalBase - (recurse fileset._internalBase fileset._internalTree); + (fromDir fileset._internalBase fileset._internalTree); } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 4247c71efcf..4ddfe3a82a4 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -857,6 +857,26 @@ checkFileset 'union ./c/a (fileFilter (file: assert file.name != "a"; true) ./.) # but here we need to use ./c checkFileset 'union (fileFilter (file: assert file.name != "a"; true) ./.) ./c' +# Also lazy, the filter isn't called on a filtered out path +tree=( + [a]=1 + [b]=0 + [c]=0 +) +checkFileset 'fileFilter (file: assert file.name != "c"; file.name == "a") (difference ./. ./c)' + +# Make sure single files are filtered correctly +tree=( + [a]=1 + [b]=0 +) +checkFileset 'fileFilter (file: assert file.name == "a"; true) ./a' +tree=( + [a]=0 + [b]=0 +) +checkFileset 'fileFilter (file: assert file.name == "a"; false) ./a' + ## Tracing # The second trace argument is returned -- cgit 1.4.1 From 66261e9961254a5c8a7c096e091fd42deb988cee Mon Sep 17 00:00:00 2001 From: Felix Buehler Date: Sun, 25 Jun 2023 13:16:06 +0200 Subject: lib.lists.allUnique: init --- lib/default.nix | 2 +- lib/lists.nix | 13 +++++++++++++ lib/tests/misc.nix | 9 +++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/default.nix b/lib/default.nix index fe737a125e6..80fc1798587 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -92,7 +92,7 @@ let concatMap flatten remove findSingle findFirst any all count optional optionals toList range replicate partition zipListsWith zipLists reverseList listDfs toposort sort naturalSort compareLists take - drop sublist last init crossLists unique intersectLists + drop sublist last init crossLists unique allUnique intersectLists subtractLists mutuallyExclusive groupBy groupBy'; inherit (self.strings) concatStrings concatMapStrings concatImapStrings intersperse concatStringsSep concatMapStringsSep diff --git a/lib/lists.nix b/lib/lists.nix index 3835e3ba69c..15047f488f4 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -821,6 +821,19 @@ rec { */ unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; + /* Check if list contains only unique elements. O(n^2) complexity. + + Type: allUnique :: [a] -> bool + + Example: + allUnique [ 3 2 3 4 ] + => false + allUnique [ 3 2 4 1 ] + => true + */ + allUnique = list: (length (unique list) == length list); + + /* Intersects list 'e' and another list. O(nm) complexity. Example: diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 0d30e93aafb..06cb5e763e2 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -726,6 +726,15 @@ runTests { expected = 7; }; + testAllUnique_true = { + expr = allUnique [ 3 2 4 1 ]; + expected = true; + }; + testAllUnique_false = { + expr = allUnique [ 3 2 3 4 ]; + expected = false; + }; + # ATTRSETS testConcatMapAttrs = { -- cgit 1.4.1 From 1c3eb9eff1b864bf49c3661558b495235fc3b3b4 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 14 Nov 2023 07:42:59 +0100 Subject: lib.fileset.fileFilter: Restrict second argument to paths While this change is backwards-incompatible, I think it's okay because: - The `fileFilter` function is not yet in a stable NixOS release, it was only merged about [a month ago](https://github.com/NixOS/nixpkgs/pull/257356). - All public uses of the function on GitHub only pass a path - Any `fileFilter pred fileset` can also be expressed as `intersection fileset (fileFilter pred path)` without loss of functionality. - This is furthermore pointed out in the new error message when a file set is passed --- lib/fileset/README.md | 15 +++++++++++++++ lib/fileset/default.nix | 20 +++++++++++++++----- lib/fileset/internal.nix | 33 ++++++++++++++++++--------------- lib/fileset/tests.sh | 15 ++++----------- 4 files changed, 52 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/fileset/README.md b/lib/fileset/README.md index 91f892a1be9..14b6877a906 100644 --- a/lib/fileset/README.md +++ b/lib/fileset/README.md @@ -238,6 +238,21 @@ Arguments: And it would be unclear how the library should behave if the one file wouldn't be added to the store: `toSource { root = ./file.nix; fileset = ; }` has no reasonable result because returing an empty store path wouldn't match the file type, and there's no way to have an empty file store path, whatever that would mean. +### `fileFilter` takes a path + +The `fileFilter` function takes a path, and not a file set, as its second argument. + +- (-) Makes it harder to compose functions, since the file set type, the return value, can't be passed to the function itself like `fileFilter predicate fileset` + - (+) It's still possible to use `intersection` to filter on file sets: `intersection fileset (fileFilter predicate ./.)` + - (-) This does need an extra `./.` argument that's not obvious + - (+) This could always be `/.` or the project directory, `intersection` will make it lazy +- (+) In the future this will allow `fileFilter` to support a predicate property like `subpath` and/or `components` in a reproducible way. + This wouldn't be possible if it took a file set, because file sets don't have a predictable absolute path. + - (-) What about the base path? + - (+) That can change depending on which files are included, so if it's used for `fileFilter` + it would change the `subpath`/`components` value depending on which files are included. +- (+) If necessary, this restriction can be relaxed later, the opposite wouldn't be possible + ## To update in the future Here's a list of places in the library that need to be updated in the future: diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index d90c770633d..81be1af9d8a 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -366,7 +366,7 @@ in { type :: String, ... } -> Bool) - -> FileSet + -> Path -> FileSet Example: @@ -397,14 +397,24 @@ in { Other attributes may be added in the future. */ predicate: - # The file set to filter based on the predicate function - fileset: + # The path whose files to filter + path: if ! isFunction predicate then throw '' lib.fileset.fileFilter: First argument is of type ${typeOf predicate}, but it should be a function instead.'' + else if ! isPath path then + if path._type or "" == "fileset" then + throw '' + lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. + If you need to filter files in a file set, use `intersection fileset (fileFilter pred ./.)` instead.'' + else + throw '' + lib.fileset.fileFilter: Second argument is of type ${typeOf path}, but it should be a path instead.'' + else if ! pathExists path then + throw '' + lib.fileset.fileFilter: Second argument (${toString path}) is a path that does not exist.'' else - _fileFilter predicate - (_coerce "lib.fileset.fileFilter: Second argument" fileset); + _fileFilter predicate path; /* The file set containing all files that are in both of two given file sets. diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index b245caade1f..6b5cea066af 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -786,9 +786,9 @@ rec { _differenceTree (path + "/${name}") lhsValue (rhs.${name} or null) ) (_directoryEntries path lhs); - # Filters all files in a file set based on a predicate - # Type: ({ name, type, ... } -> Bool) -> FileSet -> FileSet - _fileFilter = predicate: fileset: + # Filters all files in a path based on a predicate + # Type: ({ name, type, ... } -> Bool) -> Path -> FileSet + _fileFilter = predicate: root: let # Check the predicate for a single file # Type: String -> String -> filesetTree @@ -807,19 +807,22 @@ rec { # Check the predicate for all files in a directory # Type: Path -> filesetTree - fromDir = path: tree: - mapAttrs (name: subtree: - if isAttrs subtree || subtree == "directory" then - fromDir (path + "/${name}") subtree - else if subtree == null then - null + fromDir = path: + mapAttrs (name: type: + if type == "directory" then + fromDir (path + "/${name}") else - fromFile name subtree - ) (_directoryEntries path tree); + fromFile name type + ) (readDir path); + + rootType = pathType root; in - if fileset._internalIsEmptyWithoutBase then - _emptyWithoutBase + if rootType == "directory" then + _create root (fromDir root) else - _create fileset._internalBase - (fromDir fileset._internalBase fileset._internalTree); + # Single files are turned into a directory containing that file or nothing. + _create (dirOf root) { + ${baseNameOf root} = + fromFile (baseNameOf root) rootType; + }; } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index 5ef155d25a8..ebf9b6c37bf 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -813,14 +813,15 @@ checkFileset 'difference ./. ./b' # The first argument needs to be a function expectFailure 'fileFilter null (abort "this is not needed")' 'lib.fileset.fileFilter: First argument is of type null, but it should be a function instead.' -# The second argument can be a file set or an existing path -expectFailure 'fileFilter (file: abort "this is not needed") null' 'lib.fileset.fileFilter: Second argument is of type null, but it should be a file set or a path instead.' +# The second argument needs to be an existing path +expectFailure 'fileFilter (file: abort "this is not needed") _emptyWithoutBase' 'lib.fileset.fileFilter: Second argument is a file set, but it should be a path instead. +\s*If you need to filter files in a file set, use `intersection fileset \(fileFilter pred \./\.\)` instead.' +expectFailure 'fileFilter (file: abort "this is not needed") null' 'lib.fileset.fileFilter: Second argument is of type null, but it should be a path instead.' expectFailure 'fileFilter (file: abort "this is not needed") ./a' 'lib.fileset.fileFilter: Second argument \('"$work"'/a\) is a path that does not exist.' # The predicate is not called when there's no files tree=() checkFileset 'fileFilter (file: abort "this is not needed") ./.' -checkFileset 'fileFilter (file: abort "this is not needed") _emptyWithoutBase' # The predicate must be able to handle extra attributes touch a @@ -882,14 +883,6 @@ checkFileset 'union ./c/a (fileFilter (file: assert file.name != "a"; true) ./.) # but here we need to use ./c checkFileset 'union (fileFilter (file: assert file.name != "a"; true) ./.) ./c' -# Also lazy, the filter isn't called on a filtered out path -tree=( - [a]=1 - [b]=0 - [c]=0 -) -checkFileset 'fileFilter (file: assert file.name != "c"; file.name == "a") (difference ./. ./c)' - # Make sure single files are filtered correctly tree=( [a]=1 -- cgit 1.4.1 From 2dfb1d36cf5ff1fc9b39292ff202e895f92eae0c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Nov 2023 01:24:55 +0100 Subject: lib.fileset.gitTracked/gitTrackedWith: init A configuration parameter for gitTrackedWith will be introduced in the next commit --- lib/fileset/default.nix | 88 ++++++++++++++++++++++++++++ lib/fileset/internal.nix | 19 ++++++ lib/fileset/tests.sh | 147 ++++++++++++++++++++++++++++++++++++++++++++++- lib/tests/release.nix | 2 + 4 files changed, 255 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 81be1af9d8a..0248c16c645 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -12,6 +12,7 @@ let _printFileset _intersection _difference + _mirrorStorePath ; inherit (builtins) @@ -596,4 +597,91 @@ in { # We could also return the original fileset argument here, # but that would then duplicate work for consumers of the fileset, because then they have to coerce it again actualFileset; + + /* + Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository. + + This function behaves like [`gitTrackedWith { }`](#function-library-lib.fileset.gitTrackedWith) - using the defaults. + + Type: + gitTracked :: Path -> FileSet + + Example: + # Include all files tracked by the Git repository in the current directory + gitTracked ./. + + # Include only files tracked by the Git repository in the parent directory + # that are also in the current directory + intersection ./. (gitTracked ../.) + */ + gitTracked = + /* + The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository. + This directory must contain a `.git` file or subdirectory. + */ + path: + # See the gitTrackedWith implementation for more explanatory comments + let + fetchResult = builtins.fetchGit path; + in + if ! isPath path then + throw "lib.fileset.gitTracked: Expected the argument to be a path, but it's a ${typeOf path} instead." + else if ! pathExists (path + "/.git") then + throw "lib.fileset.gitTracked: Expected the argument (${toString path}) to point to a local working tree of a Git repository, but it's not." + else + _mirrorStorePath path fetchResult.outPath; + + /* + Create a file set containing all [Git-tracked files](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) in a repository. + The first argument allows configuration with an attribute set, + while the second argument is the path to the Git working tree. + If you don't need the configuration, + you can use [`gitTracked`](#function-library-lib.fileset.gitTracked) instead. + + This is equivalent to the result of [`unions`](#function-library-lib.fileset.unions) on all files returned by [`git ls-files`](https://git-scm.com/docs/git-ls-files) + (which uses [`--cached`](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt--c) by default). + + :::{.warning} + Currently this function is based on [`builtins.fetchGit`](https://nixos.org/manual/nix/stable/language/builtins.html#builtins-fetchGit) + As such, this function causes all Git-tracked files to be unnecessarily added to the Nix store, + without being re-usable by [`toSource`](#function-library-lib.fileset.toSource). + + This may change in the future. + ::: + + Type: + gitTrackedWith :: { } -> Path -> FileSet + + Example: + # Include all files tracked by the Git repository in the current directory + gitTracked { } ./. + */ + gitTrackedWith = + { + }: + /* + The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository. + This directory must contain a `.git` file or subdirectory. + */ + path: + let + # This imports the files unnecessarily, which currently can't be avoided + # because `builtins.fetchGit` is the only function exposing which files are tracked by Git. + # With the [lazy trees PR](https://github.com/NixOS/nix/pull/6530), + # the unnecessarily import could be avoided. + # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944). + fetchResult = builtins.fetchGit { + url = path; + }; + in + if ! isPath path then + throw "lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it's a ${typeOf path} instead." + # We can identify local working directories by checking for .git, + # see https://git-scm.com/docs/gitrepository-layout#_description. + # Note that `builtins.fetchGit` _does_ work for bare repositories (where there's no `.git`), + # even though `git ls-files` wouldn't return any files in that case. + else if ! pathExists (path + "/.git") then + throw "lib.fileset.gitTrackedWith: Expected the second argument (${toString path}) to point to a local working tree of a Git repository, but it's not." + else + _mirrorStorePath path fetchResult.outPath; } diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 6b5cea066af..93f3d5e18b9 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -825,4 +825,23 @@ rec { ${baseNameOf root} = fromFile (baseNameOf root) rootType; }; + + # Mirrors the contents of a Nix store path relative to a local path as a file set. + # Some notes: + # - The store path is read at evaluation time. + # - The store path must not include files that don't exist in the respective local path. + # + # Type: Path -> String -> FileSet + _mirrorStorePath = localPath: storePath: + let + recurse = focusedStorePath: + mapAttrs (name: type: + if type == "directory" then + recurse (focusedStorePath + "/${name}") + else + type + ) (builtins.readDir focusedStorePath); + in + _create localPath + (recurse storePath); } diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index ebf9b6c37bf..c7991e06bdf 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -95,8 +95,9 @@ expectEqual() { # Usage: expectStorePath NIX expectStorePath() { local expr=$1 - if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace \ + if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace 2>"$tmp"/stderr \ --expr "$prefixExpression ($expr)"); then + cat "$tmp/stderr" >&2 die "$expr failed to evaluate, but it was expected to succeed" fi # This is safe because we assume to get back a store path in a string @@ -1251,6 +1252,150 @@ expectEqual 'trace (intersection ./a (fromSource (lib.cleanSourceWith { }))) null' 'trace ./a/b null' rm -rf -- * +## lib.fileset.gitTracked/gitTrackedWith + +# The first/second argument has to be a path +expectFailure 'gitTracked null' 'lib.fileset.gitTracked: Expected the argument to be a path, but it'\''s a null instead.' +expectFailure 'gitTrackedWith {} null' 'lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it'\''s a null instead.' + +# The path has to contain a .git directory +expectFailure 'gitTracked ./.' 'lib.fileset.gitTracked: Expected the argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.' +expectFailure 'gitTrackedWith {} ./.' 'lib.fileset.gitTrackedWith: Expected the second argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.' + +# Checks that `gitTrackedWith` contains the same files as `git ls-files` +# for the current working directory. +# If --recurse-submodules is passed, the flag is passed through to `git ls-files` +# and as `recurseSubmodules` to `gitTrackedWith` +checkGitTrackedWith() { + # All files listed by `git ls-files` + expectedFiles=() + while IFS= read -r -d $'\0' file; do + # If there are submodules but --recurse-submodules isn't passed, + # `git ls-files` lists them as empty directories, + # we need to filter that out since we only want to check/count files + if [[ -f "$file" ]]; then + expectedFiles+=("$file") + fi + done < <(git ls-files -z) + + storePath=$(expectStorePath 'toSource { root = ./.; fileset = gitTrackedWith { } ./.; }') + + # Check that each expected file is also in the store path with the same content + for expectedFile in "${expectedFiles[@]}"; do + if [[ ! -e "$storePath"/"$expectedFile" ]]; then + die "Expected file $expectedFile to exist in $storePath, but it doesn't.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")" + fi + if ! diff "$expectedFile" "$storePath"/"$expectedFile"; then + die "Expected file $expectedFile to have the same contents as in $storePath, but it doesn't.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")" + fi + done + + # This is a cheap way to verify the inverse: That all files in the store path are also expected + # We just count the number of files in both and verify they're the same + actualFileCount=$(find "$storePath" -type f -printf . | wc -c) + if [[ "${#expectedFiles[@]}" != "$actualFileCount" ]]; then + die "Expected ${#expectedFiles[@]} files in $storePath, but got $actualFileCount.\nGit status:\n$(git status)\nStore path contents:\n$(find "$storePath")" + fi +} + + +# Runs checkGitTrackedWith, this will make more sense in the next commit +checkGitTracked() { + checkGitTrackedWith +} + +createGitRepo() { + git init -q "$1" + # Only repo-local config + git -C "$1" config user.name "Nixpkgs" + git -C "$1" config user.email "nixpkgs@nixos.org" + # Get at least a HEAD commit, needed for older Nix versions + git -C "$1" commit -q --allow-empty -m "Empty commit" +} + +# Go through all stages of Git files +# See https://www.git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository + +# Empty repository +createGitRepo . +checkGitTracked + +# Untracked file +echo a > a +checkGitTracked + +# Staged file +git add a +checkGitTracked + +# Committed file +git commit -q -m "Added a" +checkGitTracked + +# Edited file +echo b > a +checkGitTracked + +# Removed file +git rm -f -q a +checkGitTracked + +rm -rf -- * + +# gitignored file +createGitRepo . +echo a > .gitignore +touch a +git add -A +checkGitTracked + +# Add it regardless (needs -f) +git add -f a +checkGitTracked +rm -rf -- * + +# Directory +createGitRepo . +mkdir -p d1/d2/d3 +touch d1/d2/d3/a +git add d1 +checkGitTracked +rm -rf -- * + +# Submodules +createGitRepo . +createGitRepo sub + +# Untracked submodule +git -C sub commit -q --allow-empty -m "Empty commit" +checkGitTracked + +# Tracked submodule +git submodule add ./sub sub >/dev/null +checkGitTracked + +# Untracked file +echo a > sub/a +checkGitTracked + +# Staged file +git -C sub add a +checkGitTracked + +# Committed file +git -C sub commit -q -m "Add a" +checkGitTracked + +# Changed file +echo b > sub/b +checkGitTracked + +# Removed file +git -C sub rm -f -q a +checkGitTracked + +rm -rf -- * + # TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets echo >&2 tests ok diff --git a/lib/tests/release.nix b/lib/tests/release.nix index c8d6b810122..6e5b0711736 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -25,11 +25,13 @@ let ]; nativeBuildInputs = [ nix + pkgs.gitMinimal ] ++ lib.optional pkgs.stdenv.isLinux pkgs.inotify-tools; strictDeps = true; } '' datadir="${nix}/share" export TEST_ROOT=$(pwd)/test-tmp + export HOME=$(mktemp -d) export NIX_BUILD_HOOK= export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -- cgit 1.4.1 From d33f1a62f5bd8b5187632b8f562d62a39540da89 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Nov 2023 01:32:02 +0100 Subject: lib.fileset.gitTrackedWith: Introduce recurseSubmodules parameter --- lib/fileset/default.nix | 26 +++++++++++++++++++++++--- lib/fileset/internal.nix | 4 ++++ lib/fileset/tests.sh | 29 ++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 0248c16c645..1ccd3013ce5 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -13,14 +13,17 @@ let _intersection _difference _mirrorStorePath + _fetchGitSubmodulesMinver ; inherit (builtins) + isBool isList isPath pathExists seq typeOf + nixVersion ; inherit (lib.lists) @@ -35,6 +38,7 @@ let inherit (lib.strings) isStringLike + versionOlder ; inherit (lib.filesystem) @@ -650,14 +654,21 @@ in { ::: Type: - gitTrackedWith :: { } -> Path -> FileSet + gitTrackedWith :: { recurseSubmodules :: Bool ? false } -> Path -> FileSet Example: # Include all files tracked by the Git repository in the current directory - gitTracked { } ./. + # and any submodules under it + gitTracked { recurseSubmodules = true; } ./. */ gitTrackedWith = { + /* + (optional, default: `false`) Whether to recurse into [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) to also include their tracked files. + + If `true`, this is equivalent to passing the [--recurse-submodules](https://git-scm.com/docs/git-ls-files#Documentation/git-ls-files.txt---recurse-submodules) flag to `git ls-files`. + */ + recurseSubmodules ? false, }: /* The [path](https://nixos.org/manual/nix/stable/language/values#type-path) to the working directory of a local Git repository. @@ -672,9 +683,18 @@ in { # However a simpler alternative still would be [a builtins.gitLsFiles](https://github.com/NixOS/nix/issues/2944). fetchResult = builtins.fetchGit { url = path; + + # This is the only `fetchGit` parameter that makes sense in this context. + # We can't just pass `submodules = recurseSubmodules` here because + # this would fail for Nix versions that don't support `submodules`. + ${if recurseSubmodules then "submodules" else null} = true; }; in - if ! isPath path then + if ! isBool recurseSubmodules then + throw "lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it's a ${typeOf recurseSubmodules} instead." + else if recurseSubmodules && versionOlder nixVersion _fetchGitSubmodulesMinver then + throw "lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version ${_fetchGitSubmodulesMinver} and after, but Nix version ${nixVersion} is used." + else if ! isPath path then throw "lib.fileset.gitTrackedWith: Expected the second argument to be a path, but it's a ${typeOf path} instead." # We can identify local working directories by checking for .git, # see https://git-scm.com/docs/gitrepository-layout#_description. diff --git a/lib/fileset/internal.nix b/lib/fileset/internal.nix index 93f3d5e18b9..0769e654c8f 100644 --- a/lib/fileset/internal.nix +++ b/lib/fileset/internal.nix @@ -826,6 +826,10 @@ rec { fromFile (baseNameOf root) rootType; }; + # Support for `builtins.fetchGit` with `submodules = true` was introduced in 2.4 + # https://github.com/NixOS/nix/commit/55cefd41d63368d4286568e2956afd535cb44018 + _fetchGitSubmodulesMinver = "2.4"; + # Mirrors the contents of a Nix store path relative to a local path as a file set. # Some notes: # - The store path is read at evaluation time. diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index c7991e06bdf..ef8bee9e66c 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -1262,11 +1262,30 @@ expectFailure 'gitTrackedWith {} null' 'lib.fileset.gitTrackedWith: Expected the expectFailure 'gitTracked ./.' 'lib.fileset.gitTracked: Expected the argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.' expectFailure 'gitTrackedWith {} ./.' 'lib.fileset.gitTrackedWith: Expected the second argument \('"$work"'\) to point to a local working tree of a Git repository, but it'\''s not.' +# recurseSubmodules has to be a boolean +expectFailure 'gitTrackedWith { recurseSubmodules = null; } ./.' 'lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it'\''s a null instead.' + +# recurseSubmodules = true is not supported on all Nix versions +if [[ "$(nix-instantiate --eval --expr "$prefixExpression (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then + fetchGitSupportsSubmodules=1 +else + fetchGitSupportsSubmodules= + expectFailure 'gitTrackedWith { recurseSubmodules = true; } ./.' 'lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version 2.4 and after, but Nix version [0-9.]+ is used.' +fi + # Checks that `gitTrackedWith` contains the same files as `git ls-files` # for the current working directory. # If --recurse-submodules is passed, the flag is passed through to `git ls-files` # and as `recurseSubmodules` to `gitTrackedWith` checkGitTrackedWith() { + if [[ "${1:-}" == "--recurse-submodules" ]]; then + gitLsFlags="--recurse-submodules" + gitTrackedArg="{ recurseSubmodules = true; }" + else + gitLsFlags="" + gitTrackedArg="{ }" + fi + # All files listed by `git ls-files` expectedFiles=() while IFS= read -r -d $'\0' file; do @@ -1276,9 +1295,9 @@ checkGitTrackedWith() { if [[ -f "$file" ]]; then expectedFiles+=("$file") fi - done < <(git ls-files -z) + done < <(git ls-files -z $gitLsFlags) - storePath=$(expectStorePath 'toSource { root = ./.; fileset = gitTrackedWith { } ./.; }') + storePath=$(expectStorePath 'toSource { root = ./.; fileset = gitTrackedWith '"$gitTrackedArg"' ./.; }') # Check that each expected file is also in the store path with the same content for expectedFile in "${expectedFiles[@]}"; do @@ -1299,9 +1318,13 @@ checkGitTrackedWith() { } -# Runs checkGitTrackedWith, this will make more sense in the next commit +# Runs checkGitTrackedWith with and without --recurse-submodules +# Allows testing both variants together checkGitTracked() { checkGitTrackedWith + if [[ -n "$fetchGitSupportsSubmodules" ]]; then + checkGitTrackedWith --recurse-submodules + fi } createGitRepo() { -- cgit 1.4.1 From ada680bcfa7ac29e41ebbd7d108600ae59371331 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 3 Nov 2023 22:30:52 +0100 Subject: lib.fileset.gitTracked: Better error in pure eval --- lib/fileset/default.nix | 9 ++++++-- lib/fileset/tests.sh | 58 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/fileset/default.nix b/lib/fileset/default.nix index 1ccd3013ce5..15af0813eec 100644 --- a/lib/fileset/default.nix +++ b/lib/fileset/default.nix @@ -52,6 +52,7 @@ let inherit (lib.trivial) isFunction pipe + inPureEvalMode ; in { @@ -628,7 +629,9 @@ in { let fetchResult = builtins.fetchGit path; in - if ! isPath path then + if inPureEvalMode then + throw "lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292." + else if ! isPath path then throw "lib.fileset.gitTracked: Expected the argument to be a path, but it's a ${typeOf path} instead." else if ! pathExists (path + "/.git") then throw "lib.fileset.gitTracked: Expected the argument (${toString path}) to point to a local working tree of a Git repository, but it's not." @@ -690,7 +693,9 @@ in { ${if recurseSubmodules then "submodules" else null} = true; }; in - if ! isBool recurseSubmodules then + if inPureEvalMode then + throw "lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292." + else if ! isBool recurseSubmodules then throw "lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it's a ${typeOf recurseSubmodules} instead." else if recurseSubmodules && versionOlder nixVersion _fetchGitSubmodulesMinver then throw "lib.fileset.gitTrackedWith: Setting the attribute `recurseSubmodules` to `true` is only supported for Nix version ${_fetchGitSubmodulesMinver} and after, but Nix version ${nixVersion} is used." diff --git a/lib/fileset/tests.sh b/lib/fileset/tests.sh index ef8bee9e66c..3c88ebdd055 100755 --- a/lib/fileset/tests.sh +++ b/lib/fileset/tests.sh @@ -43,15 +43,29 @@ crudeUnquoteJSON() { cut -d \" -f2 } -prefixExpression='let - lib = import ; - internal = import { - inherit lib; - }; -in -with lib; -with internal; -with lib.fileset;' +prefixExpression() { + echo 'let + lib = + (import ) + ' + if [[ "${1:-}" == "--simulate-pure-eval" ]]; then + echo ' + .extend (final: prev: { + trivial = prev.trivial // { + inPureEvalMode = true; + }; + })' + fi + echo ' + ; + internal = import { + inherit lib; + }; + in + with lib; + with internal; + with lib.fileset;' +} # Check that two nix expression successfully evaluate to the same value. # The expressions have `lib.fileset` in scope. @@ -60,7 +74,7 @@ expectEqual() { local actualExpr=$1 local expectedExpr=$2 if actualResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/actualStderr \ - --expr "$prefixExpression ($actualExpr)"); then + --expr "$(prefixExpression) ($actualExpr)"); then actualExitCode=$? else actualExitCode=$? @@ -68,7 +82,7 @@ expectEqual() { actualStderr=$(< "$tmp"/actualStderr) if expectedResult=$(nix-instantiate --eval --strict --show-trace 2>"$tmp"/expectedStderr \ - --expr "$prefixExpression ($expectedExpr)"); then + --expr "$(prefixExpression) ($expectedExpr)"); then expectedExitCode=$? else expectedExitCode=$? @@ -96,7 +110,7 @@ expectEqual() { expectStorePath() { local expr=$1 if ! result=$(nix-instantiate --eval --strict --json --read-write-mode --show-trace 2>"$tmp"/stderr \ - --expr "$prefixExpression ($expr)"); then + --expr "$(prefixExpression) ($expr)"); then cat "$tmp/stderr" >&2 die "$expr failed to evaluate, but it was expected to succeed" fi @@ -109,10 +123,16 @@ expectStorePath() { # The expression has `lib.fileset` in scope. # Usage: expectFailure NIX REGEX expectFailure() { + if [[ "$1" == "--simulate-pure-eval" ]]; then + maybePure="--simulate-pure-eval" + shift + else + maybePure="" + fi local expr=$1 local expectedErrorRegex=$2 if result=$(nix-instantiate --eval --strict --read-write-mode --show-trace 2>"$tmp/stderr" \ - --expr "$prefixExpression $expr"); then + --expr "$(prefixExpression $maybePure) $expr"); then die "$expr evaluated successfully to $result, but it was expected to fail" fi stderr=$(<"$tmp/stderr") @@ -129,12 +149,12 @@ expectTrace() { local expectedTrace=$2 nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTrace \ - --expr "$prefixExpression trace ($expr)" || true + --expr "$(prefixExpression) trace ($expr)" || true actualTrace=$(sed -n 's/^trace: //p' "$tmp/stderrTrace") nix-instantiate --eval --show-trace >/dev/null 2>"$tmp"/stderrTraceVal \ - --expr "$prefixExpression traceVal ($expr)" || true + --expr "$(prefixExpression) traceVal ($expr)" || true actualTraceVal=$(sed -n 's/^trace: //p' "$tmp/stderrTraceVal") @@ -1266,7 +1286,7 @@ expectFailure 'gitTrackedWith {} ./.' 'lib.fileset.gitTrackedWith: Expected the expectFailure 'gitTrackedWith { recurseSubmodules = null; } ./.' 'lib.fileset.gitTrackedWith: Expected the attribute `recurseSubmodules` of the first argument to be a boolean, but it'\''s a null instead.' # recurseSubmodules = true is not supported on all Nix versions -if [[ "$(nix-instantiate --eval --expr "$prefixExpression (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then +if [[ "$(nix-instantiate --eval --expr "$(prefixExpression) (versionAtLeast builtins.nixVersion _fetchGitSubmodulesMinver)")" == true ]]; then fetchGitSupportsSubmodules=1 else fetchGitSupportsSubmodules= @@ -1336,6 +1356,12 @@ createGitRepo() { git -C "$1" commit -q --allow-empty -m "Empty commit" } +# Check the error message for pure eval mode +createGitRepo . +expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTracked ./.; }' 'lib.fileset.gitTracked: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.' +expectFailure --simulate-pure-eval 'toSource { root = ./.; fileset = gitTrackedWith {} ./.; }' 'lib.fileset.gitTrackedWith: This function is currently not supported in pure evaluation mode, since it currently relies on `builtins.fetchGit`. See https://github.com/NixOS/nix/issues/9292.' +rm -rf -- * + # Go through all stages of Git files # See https://www.git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository -- cgit 1.4.1 From 54c9a08aaf98d6d3fd94dc843003360077bee4cc Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 9 Nov 2023 02:08:44 -0800 Subject: lib.tests: build nix without flaky aws-sdk-cpp The aws-sdk-cpp tests are flaky. Since pull requests to staging cause nix to be rebuilt, this means that staging PRs end up getting false CI failures due to whatever is flaky in the AWS SDK tests. Since none of our CI needs to (or should be able to) contact AWS S3, let's just omit it all. Bonus: the tests build way faster. --- lib/path/tests/default.nix | 9 ++++++--- lib/tests/nix-for-tests.nix | 17 +++++++++++++++++ lib/tests/release.nix | 5 +++-- 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 lib/tests/nix-for-tests.nix (limited to 'lib') diff --git a/lib/path/tests/default.nix b/lib/path/tests/default.nix index 50d40cdfa47..93aea798acc 100644 --- a/lib/path/tests/default.nix +++ b/lib/path/tests/default.nix @@ -6,16 +6,19 @@ overlays = []; inherit system; }, + nixVersions ? import ../../tests/nix-for-tests.nix { inherit pkgs; }, libpath ? ../.., # Random seed seed ? null, }: + pkgs.runCommand "lib-path-tests" { - nativeBuildInputs = with pkgs; [ - nix + nativeBuildInputs = [ + nixVersions.stable + ] ++ (with pkgs; [ jq bc - ]; + ]); } '' # Needed to make Nix evaluation work export TEST_ROOT=$(pwd)/test-tmp diff --git a/lib/tests/nix-for-tests.nix b/lib/tests/nix-for-tests.nix new file mode 100644 index 00000000000..69dedece387 --- /dev/null +++ b/lib/tests/nix-for-tests.nix @@ -0,0 +1,17 @@ +{ pkgs +}: + +# The aws-sdk-cpp tests are flaky. Since pull requests to staging +# cause nix to be rebuilt, this means that staging PRs end up +# getting false CI failures due to whatever is flaky in the AWS +# SDK tests. Since none of our CI needs to (or should be able to) +# contact AWS S3, let's just omit it all from the Nix that runs +# CI. Bonus: the tests build way faster. +# +# See also: https://github.com/NixOS/nix/issues/7582 + +builtins.mapAttrs (_: pkg: + if builtins.isAttrs pkg + then pkg.override { withAWS = false; } + else pkg) + pkgs.nixVersions diff --git a/lib/tests/release.nix b/lib/tests/release.nix index c8d6b810122..1286ccf385f 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -1,8 +1,9 @@ { # The pkgs used for dependencies for the testing itself # Don't test properties of pkgs.lib, but rather the lib in the parent directory pkgs ? import ../.. {} // { lib = throw "pkgs.lib accessed, but the lib tests should use nixpkgs' lib path directly!"; }, - nix ? pkgs.nix, - nixVersions ? [ pkgs.nixVersions.minimum nix pkgs.nixVersions.unstable ], + nix ? pkgs-nixVersions.stable, + nixVersions ? [ pkgs-nixVersions.minimum nix pkgs-nixVersions.unstable ], + pkgs-nixVersions ? import ./nix-for-tests.nix { inherit pkgs; }, }: let -- cgit 1.4.1