* Basic transparent mirror support in fetchurl (NIXPKGS-70). When

  fetching a file with hash HASH of type TYPE, we first try to
  download <base-url>/<type>/<hash>, where <base-url> is one of a list
  of mirrors.  For instance, given

    src = fetchurl {
      url = http://releases.mozilla.org/pub/mozilla.org/firefox/releases/2.0.0.6/source/firefox-2.0.0.6-source.tar.bz2;
      sha1 = "eb72f55e4a8bf08e8c6ef227c0ade3d068ba1082";
    };

  and the mirror list [http://nix.cs.uu.nl/dist/tarballs], we first
  try to download

    http://nix.cs.uu.nl/dist/tarballs/sha1/eb72f55e4a8bf08e8c6ef227c0ade3d068ba1082

  and if that fails, we use the original URL.

  The list of mirrors is not yet user-configurable.

* `fetchurl' now also accepts an argument `urls' instead of `url' for
  a list of alternative download locations, which fetchurl will try in
  sequence.

svn path=/nixpkgs/trunk/; revision=9190

2 files changed, 99 insertions, 17 deletions

diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh
index 19514ac8667..c6ccf708855 100644
--- a/pkgs/build-support/fetchurl/builder.sh
+++ b/pkgs/build-support/fetchurl/builder.sh
@@ -1,20 +1,73 @@
 source $stdenv/setup
 
-header "downloading $out from $url"
+header "downloading file $name with $outputHashAlgo hash $outputHash..."
 
-curl --fail --location --max-redirs 20 --disable-epsv \
-    --cookie-jar cookies "$url" > "$out"
+# Curl flags to handle redirects, not use EPSV, handle cookies for
+# servers to need them during redirects, and work on SSL without a
+# certificate (this isn't a security problem because we check the
+# cryptographic hash of the output anyway). 
+curl="curl \
+ --location --max-redirs 20 \
+ --disable-epsv \
+ --cookie-jar cookies \
+ --insecure"
 
-if test "$NIX_OUTPUT_CHECKED" != "1"; then
-    if test "$outputHashAlgo" != "md5"; then
-        echo "hashes other than md5 are unsupported in Nix <= 0.7, upgrade to Nix 0.8"
-        exit 1
+
+tryDownload() {
+    local url="$1"
+    echo
+    header "trying $url"
+    success=
+    if $curl --fail "$url" --output "$out"; then
+        success=1
     fi
-    actual=$(md5sum -b "$out" | cut -c1-32)
-    if test "$actual" != "$id"; then
-        echo "hash is $actual, expected $id"
-        exit 1
+    stopNest
+}
+
+
+finish() {
+    # On old versions of Nix, verify the hash of the output.  On newer
+    # versions, Nix verifies the hash itself.
+    if test "$NIX_OUTPUT_CHECKED" != "1"; then
+        if test "$outputHashAlgo" != "md5"; then
+            echo "hashes other than md5 are unsupported in Nix <= 0.7, upgrade to Nix 0.8"
+            exit 1
+        fi
+        actual=$(md5sum -b "$out" | cut -c1-32)
+        if test "$actual" != "$id"; then
+            echo "hash is $actual, expected $id"
+            exit 1
+        fi
+    fi
+
+    stopNest
+    exit 0
+}
+
+
+for mirror in $hashedMirrors; do
+    url="$mirror/$outputHashAlgo/$outputHash"
+    if $curl --fail --silent --show-error --head "$url" \
+        --write-out "%{http_code}" --output /dev/null > code 2> log; then
+        tryDownload "$url"
+        if test -n "$success"; then finish; fi
+    else
+        # Be quiet about 404 errors, which we interpret as the file
+        # not being present on this particular mirror.
+        if test "$(cat code)" != 404; then
+            echo "error checking the existence of $url:"
+            cat log
+        fi
     fi
-fi
+done
+
+
+success=
+for url in $urls; do
+    tryDownload "$url"
+    if test -n "$success"; then finish; fi
+done
+
 
-stopNest
+echo "error: cannot download $name from any mirror"
+exit 1
diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix
index b6eec09c13b..c79f3d214c6 100644
--- a/pkgs/build-support/fetchurl/default.nix
+++ b/pkgs/build-support/fetchurl/default.nix
@@ -3,16 +3,47 @@
 
 {stdenv, curl}: # Note that `curl' may be `null', in case of the native stdenv.
 
-{name ? "", url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? ""}:
+{ # URL to fetch.
+  url ? ""
+
+, # Alternatively, a list of URLs specifying alternative download
+  # locations.  They are tried in order.
+  urls ? []
+
+, # Name of the file.  If empty, use the basename of `url' (or of the
+  # first element of `urls').
+  name ? ""
+
+  # Different ways of specifying the hash.
+, outputHash ? ""
+, outputHashAlgo ? ""
+, md5 ? ""
+, sha1 ? ""
+, sha256 ? ""
+}:
+
+assert urls != [] -> url == "";
+assert url != "" -> urls == [];
 
 assert (outputHash != "" && outputHashAlgo != "")
     || md5 != "" || sha1 != "" || sha256 != "";
 
+let urls_ = if urls != [] then urls else [url]; in
+
 stdenv.mkDerivation {
-  name = if name != "" then name else baseNameOf (toString url);
+  name =
+    if name != "" then name
+    else baseNameOf (toString (builtins.head urls_));
   builder = ./builder.sh;
   buildInputs = [curl];
 
+  urls = urls_;
+
+  # The content-addressable mirrors.
+  hashedMirrors = [
+    http://nix.cs.uu.nl/dist/tarballs
+  ];
+
   # Compatibility with Nix <= 0.7.
   id = md5;
 
@@ -22,8 +53,6 @@ stdenv.mkDerivation {
   outputHash = if outputHash != "" then outputHash else
       if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
   
-  inherit url;
-
   # We borrow these environment variables from the caller to allow
   # easy proxy configuration.  This is impure, but a fixed-output
   # derivation like fetchurl is allowed to do so since its result is