copy-tarballs.pl: Revive extracting all tarballs from release.nix

6 files changed, 47 insertions, 9 deletions

diff --git a/maintainers/scripts/all-tarballs.nix b/maintainers/scripts/all-tarballs.nix
new file mode 100644
index 00000000000..552f8802296
--- /dev/null
+++ b/maintainers/scripts/all-tarballs.nix
@@ -0,0 +1,18 @@
+/* Helper expression for copy-tarballs. This returns (nearly) all
+   tarballs used the free packages in Nixpkgs.
+
+   Typical usage:
+
+   $ copy-tarballs.pl --expr 'import <nixpkgs/maintainers/scripts/all-tarballs.nix>'
+*/
+
+removeAttrs (import ../../pkgs/top-level/release.nix
+  { # Don't apply ‘hydraJob’ to jobs, because then we can't get to the
+    # dependency graph.
+    scrubJobs = false;
+    # No need to evaluate on i686.
+    supportedSystems = [ "x86_64-linux" ];
+  })
+  [ # Remove jobs whose evaluation depends on a writable Nix store.
+    "tarball" "unstable"
+  ]
diff --git a/maintainers/scripts/copy-tarballs.pl b/maintainers/scripts/copy-tarballs.pl
index 08a8638e25a..8ce982122c3 100755
--- a/maintainers/scripts/copy-tarballs.pl
+++ b/maintainers/scripts/copy-tarballs.pl
@@ -93,7 +93,7 @@ elsif ($op eq "--expr") {
 
     # Evaluate find-tarballs.nix.
     my $expr = $ARGV[0] // die "$0: --expr requires a Nix expression\n";
-    my $pid = open(JSON, "-|", "nix-instantiate", "--eval-only", "--json", "--strict",
+    my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
                    "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
                    "--arg", "expr", $expr);
     my $stdout = <JSON>;
@@ -113,6 +113,11 @@ elsif ($op eq "--expr") {
         my $algo = $fetch->{type};
         my $hash = $fetch->{hash};
 
+        if (defined $ENV{DEBUG}) {
+            print "$url $algo $hash\n";
+            next;
+        }
+
         if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
             print STDERR "skipping $url (unsupported scheme)\n";
             next;
diff --git a/maintainers/scripts/find-tarballs.nix b/maintainers/scripts/find-tarballs.nix
index 2dd17d9df88..e989c0e2a9e 100644
--- a/maintainers/scripts/find-tarballs.nix
+++ b/maintainers/scripts/find-tarballs.nix
@@ -1,10 +1,9 @@
-# This expression returns a list of all fetchurl calls used by all
-# packages reachable from release.nix.
+# This expression returns a list of all fetchurl calls used by ‘expr’.
 
 with import ../.. { };
 with lib;
 
-{ expr ? removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ] }:
+{ expr }:
 
 let
 
diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix
index fdd8fb0ef7d..c14f94d95fd 100644
--- a/pkgs/top-level/make-tarball.nix
+++ b/pkgs/top-level/make-tarball.nix
@@ -14,7 +14,7 @@ releaseTools.sourceTarball rec {
   version = builtins.readFile ../../.version;
   versionSuffix = "pre${toString nixpkgs.revCount}.${nixpkgs.shortRev}";
 
-  buildInputs = [ nix ];
+  buildInputs = [ nix jq ];
 
   configurePhase = ''
     eval "$preConfigure"
@@ -83,7 +83,15 @@ releaseTools.sourceTarball rec {
     stopNest
 
     header "checking find-tarballs.nix"
-    nix-instantiate --eval --strict --show-trace ./maintainers/scripts/find-tarballs.nix > /dev/null
+    nix-instantiate --eval --strict --show-trace --json \
+       ./maintainers/scripts/find-tarballs.nix \
+      --arg expr 'import ./maintainers/scripts/all-tarballs.nix' > $TMPDIR/tarballs.json
+    nrUrls=$(jq -r '.[].url' < $TMPDIR/tarballs.json | wc -l)
+    echo "found $nrUrls URLs"
+    if [ "$nrUrls" -lt 10000 ]; then
+      echo "suspiciously low number of URLs"
+      exit 1
+    fi
     stopNest
   '';
 
diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix
index 15380ea43c8..2f0296223a0 100644
--- a/pkgs/top-level/release-lib.nix
+++ b/pkgs/top-level/release-lib.nix
@@ -1,4 +1,8 @@
-{ supportedSystems, packageSet ? (import ./all-packages.nix), allowTexliveBuilds ? false }:
+{ supportedSystems
+, packageSet ? (import ./all-packages.nix)
+, allowTexliveBuilds ? false
+, scrubJobs ? true
+}:
 
 with import ../../lib;
 
@@ -14,6 +18,9 @@ rec {
   pkgs = pkgsFor "x86_64-linux";
 
 
+  hydraJob' = if scrubJobs then hydraJob else id;
+
+
   /* !!! Hack: poor man's memoisation function.  Necessary to prevent
      Nixpkgs from being evaluated again and again for every
      job/platform pair. */
@@ -48,7 +55,7 @@ rec {
      a derivation for each supported platform, i.e. ‘{ x86_64-linux =
      f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */
   testOn = systems: f: genAttrs
-    (filter (x: elem x supportedSystems) systems) (system: hydraJob (f (pkgsFor system)));
+    (filter (x: elem x supportedSystems) systems) (system: hydraJob' (f (pkgsFor system)));
 
 
   /* Similar to the testOn function, but with an additional
diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix
index d0c5faf823a..c55f8f0825f 100644
--- a/pkgs/top-level/release.nix
+++ b/pkgs/top-level/release.nix
@@ -13,9 +13,10 @@
 , officialRelease ? false
 , # The platforms for which we build Nixpkgs.
   supportedSystems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" ]
+, scrubJobs ? true
 }:
 
-with import ./release-lib.nix { inherit supportedSystems; };
+with import ./release-lib.nix { inherit supportedSystems scrubJobs; };
 
 let