1 files changed, 45 insertions, 16 deletions

diff --git a/maintainers/scripts/copy-tarballs.pl b/maintainers/scripts/copy-tarballs.pl
index 6be1300e513..b1233827ad8 100755
--- a/maintainers/scripts/copy-tarballs.pl
+++ b/maintainers/scripts/copy-tarballs.pl
@@ -1,5 +1,5 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 nixUnstable
+#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp nixUnstable
 
 # This command uploads tarballs to tarballs.nixos.org, the
 # content-addressed cache used by fetchurl as a fallback for when
@@ -17,6 +17,7 @@ use strict;
 use warnings;
 use File::Basename;
 use File::Path;
+use File::Slurp;
 use JSON;
 use Net::Amazon::S3;
 use Nix::Store;
@@ -33,9 +34,21 @@ my $s3 = Net::Amazon::S3->new(
 
 my $bucket = $s3->bucket("nixpkgs-tarballs") or die;
 
+my $cacheFile = "/tmp/copy-tarballs-cache";
+my %cache;
+$cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
+
+END() {
+    write_file($cacheFile, map { "$_\n" } keys %cache);
+}
+
 sub alreadyMirrored {
     my ($algo, $hash) = @_;
-    return defined $bucket->get_key("$algo/$hash");
+    my $key = "$algo/$hash";
+    return 1 if defined $cache{$key};
+    my $res = defined $bucket->get_key($key);
+    $cache{$key} = 1 if $res;
+    return $res;
 }
 
 sub uploadFile {
@@ -50,41 +63,52 @@ sub uploadFile {
 
     my $mainKey = "sha512/$sha512_16";
 
-    # Upload the file as sha512/<hash-in-base-16>.
-    print STDERR "uploading $fn to $mainKey...\n";
-    $bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name })
-        or die "failed to upload $fn to $mainKey\n";
-
     # Create redirects from the other hash types.
     sub redirect {
         my ($name, $dest) = @_;
         #print STDERR "linking $name to $dest...\n";
         $bucket->add_key($name, "", { 'x-amz-website-redirect-location' => "/" . $dest })
             or die "failed to create redirect from $name to $dest\n";
+        $cache{$name} = 1;
     }
     redirect "md5/$md5_16", $mainKey;
     redirect "sha1/$sha1_16", $mainKey;
     redirect "sha256/$sha256_32", $mainKey;
     redirect "sha256/$sha256_16", $mainKey;
     redirect "sha512/$sha512_32", $mainKey;
+
+    # Upload the file as sha512/<hash-in-base-16>.
+    print STDERR "uploading $fn to $mainKey...\n";
+    $bucket->add_key_filename($mainKey, $fn, { 'x-amz-meta-original-name' => $name })
+        or die "failed to upload $fn to $mainKey\n";
+    $cache{$mainKey} = 1;
 }
 
-my $op = $ARGV[0] // "";
+my $op = shift @ARGV;
 
 if ($op eq "--file") {
-    my $fn = $ARGV[1] // die "$0: --file requires a file name\n";
-    if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
-        print STDERR "$fn is already mirrored\n";
-    } else {
-        uploadFile($fn, basename $fn);
+    my $res = 0;
+    foreach my $fn (@ARGV) {
+        eval {
+            if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
+                print STDERR "$fn is already mirrored\n";
+            } else {
+                uploadFile($fn, basename $fn);
+            }
+        };
+        if ($@) {
+            warn "$@\n";
+            $res = 1;
+        }
     }
+    exit $res;
 }
 
 elsif ($op eq "--expr") {
 
     # Evaluate find-tarballs.nix.
-    my $expr = $ARGV[1] // die "$0: --expr requires a Nix expression\n";
-    my $pid = open(JSON, "-|", "nix-instantiate", "--eval-only", "--json", "--strict",
+    my $expr = $ARGV[0] // die "$0: --expr requires a Nix expression\n";
+    my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
                    "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
                    "--arg", "expr", $expr);
     my $stdout = <JSON>;
@@ -104,6 +128,11 @@ elsif ($op eq "--expr") {
         my $algo = $fetch->{type};
         my $hash = $fetch->{hash};
 
+        if (defined $ENV{DEBUG}) {
+            print "$url $algo $hash\n";
+            next;
+        }
+
         if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
             print STDERR "skipping $url (unsupported scheme)\n";
             next;
@@ -138,5 +167,5 @@ elsif ($op eq "--expr") {
 }
 
 else {
-    die "Syntax: $0 --file FILENAME | --expr EXPR\n";
+    die "Syntax: $0 --file FILENAMES... | --expr EXPR\n";
 }