Merge remote-tracking branch 'origin/master' into staging

Conflicts:
	pkgs/servers/serfdom/default.nix

13 files changed, 244 insertions, 22 deletions

diff --git a/pkgs/build-support/cabal/default.nix b/pkgs/build-support/cabal/default.nix
index 67934ed0c29..6f454dc2b58 100644
--- a/pkgs/build-support/cabal/default.nix
+++ b/pkgs/build-support/cabal/default.nix
@@ -82,9 +82,7 @@ assert !enableStaticLibraries -> versionOlder "7.7" ghc.version;
             # the default download location for Cabal packages is Hackage,
             # you still have to specify the checksum
             src = fetchurl {
-              # cannot use mirrors system because of subtly different directory structures
-              urls = ["http://hackage.haskell.org/packages/archive/${self.pname}/${self.version}/${self.fname}.tar.gz"
-                      "http://hdiff.luite.com/packages/archive/${self.pname}/${self.fname}.tar.gz"];
+              url = "mirror://hackage/${self.pname}/${self.fname}.tar.gz";
               inherit (self) sha256;
             };
 
diff --git a/pkgs/build-support/fetchbzr/builder.sh b/pkgs/build-support/fetchbzr/builder.sh
index 17567fdadd2..af1257d3688 100644
--- a/pkgs/build-support/fetchbzr/builder.sh
+++ b/pkgs/build-support/fetchbzr/builder.sh
@@ -1,9 +1,9 @@
 source "$stdenv/setup"
 
-header "exporting \`$url' (revision $revision) into \`$out'"
+header "exporting \`$url' (revision $rev) into \`$out'"
 
 # Perform a lightweight checkout so that we don't end up importing
 # all the repository's history.
-bzr -Ossl.cert_reqs=none export -r "$revision" --format=dir "$out" "$url"
+bzr -Ossl.cert_reqs=none export -r "$rev" --format=dir "$out" "$url"
 
 stopNest
diff --git a/pkgs/build-support/fetchbzr/default.nix b/pkgs/build-support/fetchbzr/default.nix
index 721250beeb3..dd2c0363187 100644
--- a/pkgs/build-support/fetchbzr/default.nix
+++ b/pkgs/build-support/fetchbzr/default.nix
@@ -1,5 +1,5 @@
 { stdenv, bazaar }: 
-{ url, revision, sha256 }:
+{ url, rev, sha256 }:
 
 stdenv.mkDerivation {
   name = "bzr-export";
@@ -11,5 +11,5 @@ stdenv.mkDerivation {
   outputHashMode = "recursive";
   outputHash = sha256;
   
-  inherit url revision;
+  inherit url rev;
 }
diff --git a/pkgs/build-support/fetchdarcs/builder.sh b/pkgs/build-support/fetchdarcs/builder.sh
index efff5ff5ebf..301deb98307 100644
--- a/pkgs/build-support/fetchdarcs/builder.sh
+++ b/pkgs/build-support/fetchdarcs/builder.sh
@@ -2,9 +2,9 @@ source $stdenv/setup
 
 tagtext=""
 tagflags=""
-if test -n "$tag"; then
-    tagtext="(tag $tag) "
-    tagflags="--tag=$tag"
+if test -n "$rev"; then
+    tagtext="(tag $rev) "
+    tagflags="--tag=$rev"
 elif test -n "$context"; then
     tagtext="(context) "
     tagflags="--context=$context"
diff --git a/pkgs/build-support/fetchdarcs/default.nix b/pkgs/build-support/fetchdarcs/default.nix
index 63e4ecde88a..3c2e0524eea 100644
--- a/pkgs/build-support/fetchdarcs/default.nix
+++ b/pkgs/build-support/fetchdarcs/default.nix
@@ -1,4 +1,4 @@
-{stdenv, darcs, nix}: {url, tag ? null, context ? null, md5 ? "", sha256 ? ""}:
+{stdenv, darcs, nix}: {url, rev ? null, context ? null, md5 ? "", sha256 ? ""}:
 
 stdenv.mkDerivation {
   name = "fetchdarcs";
@@ -9,5 +9,5 @@ stdenv.mkDerivation {
   outputHashMode = "recursive";
   outputHash = if sha256 == "" then md5 else sha256;
   
-  inherit url tag context;
+  inherit url rev context;
 }
diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git
index 3574f3b7539..bd1d8b5c1af 100755
--- a/pkgs/build-support/fetchgit/nix-prefetch-git
+++ b/pkgs/build-support/fetchgit/nix-prefetch-git
@@ -217,7 +217,9 @@ clone_user_rev() {
             fi;;
     esac
 
-    echo "git revision is $(cd $dir && (git rev-parse $rev 2> /dev/null || git rev-parse refs/heads/fetchgit) | tail -n1)"
+    local full_revision=$(cd $dir && (git rev-parse $rev 2> /dev/null || git rev-parse refs/heads/fetchgit) | tail -n1)
+    echo "git revision is $full_revision"
+    echo "git human-readable version is $(cd $dir && (git describe $full_revision 2> /dev/null || git describe --tags $full_revision 2> /dev/null || echo -- none --))"
 
     # Allow doing additional processing before .git removal
     eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
diff --git a/pkgs/build-support/fetchhg/builder.sh b/pkgs/build-support/fetchhg/builder.sh
index 9699fb69b7f..0b51ef79b23 100644
--- a/pkgs/build-support/fetchhg/builder.sh
+++ b/pkgs/build-support/fetchhg/builder.sh
@@ -1,9 +1,9 @@
 source $stdenv/setup
-header "getting $url${tag:+ ($tag)} into $out"
+header "getting $url${rev:+ ($rev)} into $out"
 
 hg clone --insecure "$url" hg-clone
 
-hg archive -q -y ${tag:+-r "$tag"} --cwd hg-clone $out
+hg archive -q -y ${rev:+-r "$rev"} --cwd hg-clone $out
 rm -f $out/.hg_archival.txt
 
 stopNest
diff --git a/pkgs/build-support/fetchhg/default.nix b/pkgs/build-support/fetchhg/default.nix
index 1aebb7ecc87..4675cbe6ec8 100644
--- a/pkgs/build-support/fetchhg/default.nix
+++ b/pkgs/build-support/fetchhg/default.nix
@@ -1,4 +1,4 @@
-{stdenv, mercurial, nix}: {name ? null, url, tag ? null, md5 ? null, sha256 ? null}:
+{stdenv, mercurial, nix}: {name ? null, url, rev ? null, md5 ? null, sha256 ? null}:
 
 # TODO: statically check if mercurial as the https support if the url starts woth https.
 stdenv.mkDerivation {
@@ -13,6 +13,6 @@ stdenv.mkDerivation {
   outputHashMode = "recursive";
   outputHash = if md5 != null then md5 else sha256;
   
-  inherit url tag;
+  inherit url rev;
   preferLocalBuild = true;
 }
diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix
index 8417499cd00..38d3353202b 100644
--- a/pkgs/build-support/fetchurl/mirrors.nix
+++ b/pkgs/build-support/fetchurl/mirrors.nix
@@ -20,6 +20,12 @@ rec {
     http://kent.dl.sourceforge.net/sourceforge/
   ];
 
+  # SourceForge.jp.
+  sourceforgejp = [
+    http://osdn.dl.sourceforge.jp/
+    http://jaist.dl.sourceforge.jp/
+  ];
+
   # GNU (http://www.gnu.org/prep/ftp.html).
   gnu = [
     # This one redirects to a (supposedly) nearby and (supposedly) up-to-date
diff --git a/pkgs/build-support/fetchzip/nix-prefetch-zip b/pkgs/build-support/fetchzip/nix-prefetch-zip
new file mode 100755
index 00000000000..76255ab3674
--- /dev/null
+++ b/pkgs/build-support/fetchzip/nix-prefetch-zip
@@ -0,0 +1,153 @@
+#! /bin/sh -e
+
+usage(){
+    echo  >&2 "syntax: nix-prefetch-zip [OPTIONS] [URL [EXPECTED-HASH]]
+
+Options:
+      --url         url    The url of the archive to fetch.
+      --name        name   The name to use for the store path (defaults to \`basename \$url\`).
+      --hash        hash   The hash of unpacked archive.
+      --hash-type   type   Use the specified cryptographic hash algorithm, which can be one of md5, sha1, and sha256.
+      --leave-root         Keep the root directory of the archive.
+      --help               Show this help text.
+"
+    exit 1
+}
+
+
+argi=0
+argfun=""
+for arg; do
+  if test -z "$argfun"; then
+    case $arg in
+      --url) argfun=set_url;;
+      --name) argfun=set_name;;
+      --hash) argfun=set_expHash;;
+      --hash-type) argfun=set_hashType;;
+      --leave-root) leaveRoot=true;;
+      --help) usage;;
+      *) argi=$(($argi + 1))
+         case $argi in
+           1) url=$arg;;
+           2) rev=$arg;;
+           3) expHash=$arg;;
+           *) echo "Unexpected argument: $arg" >&2
+              usage
+              ;;
+         esac
+         ;;
+    esac
+  else
+    case $argfun in
+      set_*)
+        var=$(echo $argfun | sed 's,^set_,,')
+        eval "$var=\$arg"
+        ;;
+    esac
+    argfun=""
+  fi
+done
+
+if [ -z "$url" ]; then
+  echo "Error: No --url flag given" >&2
+  usage
+fi
+
+if [ -z "$name" ]; then
+  name=$(basename "$url")
+fi
+
+if test -z "$hashType"; then
+  hashType=sha256
+fi
+
+hashFormat="--base32"
+
+tmp=$(mktemp -d 2>/dev/null || mktemp -d -t "$$")
+trap "rm -rf '$tmp'" EXIT
+
+unpackDir=$tmp/unpacked/$name
+mkdir -p $unpackDir
+downloadedFile=$tmp/$name
+
+unpackFile() {
+  local curSrc="$1"
+
+  case "$curSrc" in
+    *.tar.xz | *.tar.lzma)
+      # Don't rely on tar knowing about .xz.
+      xz -d < $curSrc | tar xf -
+      ;;
+    *.tar | *.tar.* | *.tgz | *.tbz2)
+      # GNU tar can automatically select the decompression method
+      # (info "(tar) gzip").
+      tar xf $curSrc
+      ;;
+    *.zip)
+      unzip -qq $curSrc
+      ;;
+    *)
+      echo "source archive $curSrc has unknown type" >&2
+      exit 1
+      ;;
+  esac
+}
+
+# If the hash was given, a file with that hash may already be in the
+# store.
+if test -n "$expHash"; then
+  finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$name")
+  if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
+    finalPath=
+  fi
+  hash=$expHash
+fi
+
+# If we don't know the hash or a path with that hash doesn't exist,
+# download the file and add it to the store.
+if test -z "$finalPath"; then
+  curl="curl \
+   --location --max-redirs 20 \
+   --disable-epsv \
+   --insecure"
+
+  if ! $curl --fail "$url" --output "$downloadedFile"; then
+    echo "error: could not download $url" >&2
+    exit 1
+  fi
+
+  cd $unpackDir
+  unpackFile "$downloadedFile"
+
+  # FIXME: handle zip files that contain a single regular file.
+  if [ -z "$leaveRoot" ]; then
+    shopt -s dotglob
+    if [ $(ls -d $unpackDir/* | wc -l) != 1 ]; then
+      echo "error: zip file must contain a single directory."
+      exit 1
+    fi
+    fn=$(cd "$unpackDir" && echo *)
+    mv $unpackDir/$fn/* "$unpackDir/"
+    rmdir "$unpackDir/$fn"
+  fi
+
+  # Compute the hash.
+  hash=$(nix-hash --type $hashType $hashFormat $unpackDir)
+  if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+
+  # Add the downloaded file to the Nix store.
+  finalPath=$(nix-store --add-fixed --recursive "$hashType" $unpackDir)
+
+  if test -n "$expHash" -a "$expHash" != "$hash"; then
+    echo "hash mismatch for URL \`$url'"
+    exit 1
+  fi
+fi
+
+if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
+
+echo $hash
+
+if test -n "$PRINT_PATH"; then
+  echo $finalPath
+fi
diff --git a/pkgs/build-support/kernel/paths-from-graph.pl b/pkgs/build-support/kernel/paths-from-graph.pl
index f1866237210..9a199a2b304 100644
--- a/pkgs/build-support/kernel/paths-from-graph.pl
+++ b/pkgs/build-support/kernel/paths-from-graph.pl
@@ -1,9 +1,27 @@
+# Parses a /nix/store/*-closure file and prints
+# various information.
+# By default, the nodes in the graph are printed to stdout.
+# If the environment variable printManifest is set,
+# then the graph is written as a manifest.
+# If printRegistration is set, then the graph is written
+# as a registration file for a manifest is written
+# in the `nix-store --load-db' format.
+
 use strict;
 use File::Basename;
 
 my %storePaths;
 my %refs;
 
+# Each argument on the command line is a graph file.
+# The graph file contains line-triples and a variable
+# number of references:
+# <store-path>
+# <deriver>
+# <count>
+# <ref-#1>
+# ...
+# <ref-#count>
 foreach my $graph (@ARGV) {
     open GRAPH, "<$graph" or die;
 
diff --git a/pkgs/build-support/upstream-updater/update-walker-service-specific.sh b/pkgs/build-support/upstream-updater/update-walker-service-specific.sh
index a979e24edf2..28c28f69587 100644
--- a/pkgs/build-support/upstream-updater/update-walker-service-specific.sh
+++ b/pkgs/build-support/upstream-updater/update-walker-service-specific.sh
@@ -5,7 +5,7 @@ SF_redirect () {
 }
 
 SF_version_dir () {
-  version_link 'http://sourceforge.net/.+/[0-9.]+/$'
+  version_link 'http://sourceforge.net/.+/'"$1"'[0-9.]+/$'
 }
 
 GH_latest () {
diff --git a/pkgs/build-support/upstream-updater/update-walker.sh b/pkgs/build-support/upstream-updater/update-walker.sh
index bc58f935f81..e11eb722e0e 100755
--- a/pkgs/build-support/upstream-updater/update-walker.sh
+++ b/pkgs/build-support/upstream-updater/update-walker.sh
@@ -2,15 +2,22 @@
 
 own_dir="$(cd "$(dirname "$0")"; pwd)"
 
+URL_WAS_SET=
+DL_URL_RE=
 CURRENT_URL=
 CURRENT_REV=
 PREFETCH_COMMAND=
 NEED_TO_CHOOSE_URL=1
 
 url () {
+  URL_WAS_SET=1
   CURRENT_URL="$1"
 }
 
+dl_url_re () {
+  DL_URL_RE="$1"
+}
+
 version_unpack () {
   sed -re '
     s/[.]/ /g; 
@@ -101,16 +108,27 @@ ensure_name () {
 
 ensure_attribute_name () {
   echo "Ensuring attribute name. CURRENT_ATTRIBUTE_NAME: $CURRENT_ATTRIBUTE_NAME" >&2
+  ensure_name
   [ -z "$CURRENT_ATTRIBUTE_NAME" ] && attribute_name "$CURRENT_NAME"
   echo "Resulting attribute name: $CURRENT_ATTRIBUTE_NAME"
 }
 
+ensure_url () {
+  echo "Ensuring starting URL. CURRENT_URL: $CURRENT_URL" >&2
+  ensure_attribute_name
+  [ -z "$CURRENT_URL" ] && CURRENT_URL="$(retrieve_meta downloadPage)"
+  [ -z "$CURRENT_URL" ] && CURRENT_URL="$(retrieve_meta downloadpage)"
+  [ -z "$CURRENT_URL" ] && CURRENT_URL="$(retrieve_meta homepage)"
+  echo "Resulting URL: $CURRENT_URL"
+}
+
 ensure_choice () {
   echo "Ensuring that choice is made." >&2
   echo "NEED_TO_CHOOSE_URL: [$NEED_TO_CHOOSE_URL]." >&2
   echo "CURRENT_URL: $CURRENT_URL" >&2
+  [ -z "$URL_WAS_SET" ] && [ -z "$CURRENT_URL" ] && ensure_url
   [ -n "$NEED_TO_CHOOSE_URL" ] && {
-    version_link '[.]tar[.]([^./])+$'
+    version_link "${DL_URL_RE:-[.]tar[.]([^./])+\$}"
     unset NEED_TO_CHOOSE_URL
   }
   [ -z "$CURRENT_URL" ] && {
@@ -153,8 +171,18 @@ attribute_name () {
   echo "CURRENT_ATTRIBUTE_NAME: $CURRENT_ATTRIBUTE_NAME" >&2
 }
 
+retrieve_meta () {
+  nix-instantiate --eval-only '<nixpkgs>' -A "$CURRENT_ATTRIBUTE_NAME".meta."$1" | xargs
+}
+
 retrieve_version () {
-  PACKAGED_VERSION="$(nix-instantiate --eval-only '<nixpkgs>' -A "$CURRENT_ATTRIBUTE_NAME".meta.version | xargs)"
+  PACKAGED_VERSION="$(retrieve_meta version)"
+}
+
+ensure_dl_url_re () {
+  echo "Ensuring DL_URL_RE. DL_URL_RE: $DL_URL_RE" >&2
+  [ -z "$DL_URL_RE" ] && dl_url_re "$(retrieve_meta downloadURLRegexp)"
+  echo "DL_URL_RE: $DL_URL_RE" >&2
 }
 
 directory_of () {
@@ -256,10 +284,27 @@ process_config () {
   CONFIG_DIR="$(directory_of "$1")"
   CONFIG_NAME="$(basename "$1")"
   BEGIN_EXPRESSION='# Generated upstream information';
-  source "$CONFIG_DIR/$CONFIG_NAME"
-  ensure_name
+  if [ -f  "$CONFIG_DIR/$CONFIG_NAME" ] &&
+      [ "${CONFIG_NAME}" = "${CONFIG_NAME%.nix}" ]; then
+    source "$CONFIG_DIR/$CONFIG_NAME"
+  else
+    CONFIG_NAME="${CONFIG_NAME%.nix}"
+    ensure_attribute_name
+    [ -n "$(retrieve_meta updateWalker)" ] ||
+        [ -n "$FORCE_UPDATE_WALKER" ] || {
+      echo "Error: package not marked as safe for update-walker" >&2
+      echo "Set FORCE_UPDATE_WALKER=1 to override" >&2
+      exit 1;
+    }
+    [ -z "$(retrieve_meta fullRegenerate)" ] && eval "
+      do_overwrite(){
+        do_overwrite_just_version
+      }
+    "
+  fi
   ensure_attribute_name
   retrieve_version
+  ensure_dl_url_re
   ensure_choice
   ensure_version
   ensure_target