Merge branch 'master' into patch-3

8 files changed, 626 insertions, 277 deletions

diff --git a/maintainers/docker/.dockerignore b/maintainers/docker/.dockerignore
deleted file mode 100644
index eb4668233e6..00000000000
--- a/maintainers/docker/.dockerignore
+++ /dev/null
@@ -1,14 +0,0 @@
-*~
-,*
-.*.swp
-.*.swo
-result
-result-*
-/doc/NEWS.html
-/doc/NEWS.txt
-/doc/manual.html
-/doc/manual.pdf
-.version-suffix
-
-.DS_Store
-.git
diff --git a/maintainers/docker/Dockerfile b/maintainers/docker/Dockerfile
deleted file mode 100644
index f02a9d653fd..00000000000
--- a/maintainers/docker/Dockerfile
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM busybox
-
-RUN dir=`mktemp -d` && trap 'rm -rf "$dir"' EXIT && \
-    wget -O- https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2  | bzcat | tar x -C $dir && \
-    mkdir -m 0755 /nix && USER=root sh $dir/*/install && \
-    echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile
-
-ADD . /root/nix/nixpkgs
-ONBUILD ENV NIX_PATH nixpkgs=/root/nix/nixpkgs:nixos=/root/nix/nixpkgs/nixos
-ONBUILD ENV PATH /root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin
-ONBUILD ENV ENV /etc/profile
-ENV ENV /etc/profile
diff --git a/maintainers/scripts/all-tarballs.nix b/maintainers/scripts/all-tarballs.nix
index 552f8802296..d981a1fa7db 100644
--- a/maintainers/scripts/all-tarballs.nix
+++ b/maintainers/scripts/all-tarballs.nix
@@ -14,5 +14,5 @@ removeAttrs (import ../../pkgs/top-level/release.nix
     supportedSystems = [ "x86_64-linux" ];
   })
   [ # Remove jobs whose evaluation depends on a writable Nix store.
-    "tarball" "unstable"
+    "tarball" "unstable" "darwin-tested"
   ]
diff --git a/maintainers/scripts/gnome.sh b/maintainers/scripts/gnome.sh
index b2d55d7fbed..e5a8d606f1a 100755
--- a/maintainers/scripts/gnome.sh
+++ b/maintainers/scripts/gnome.sh
@@ -6,7 +6,7 @@ GNOME_FTP=ftp.gnome.org/pub/GNOME/sources
 
 # projects that don't follow the GNOME major versioning, or that we don't want to
 # programmatically update
-NO_GNOME_MAJOR='gtkhtml gdm'
+NO_GNOME_MAJOR="ghex gtkhtml gdm"
 
 usage() {
   echo "Usage: $0 gnome_dir <show project>|<update project>|<update-all> [major.minor]" >&2
diff --git a/maintainers/scripts/hydra-eval-failures.py b/maintainers/scripts/hydra-eval-failures.py
index b339f296056..6bbc0a45e44 100755
--- a/maintainers/scripts/hydra-eval-failures.py
+++ b/maintainers/scripts/hydra-eval-failures.py
@@ -48,8 +48,8 @@ def get_maintainers(attr_name):
 @click.command()
 @click.option(
     '--jobset',
-    default="nixos/release-16.09",
-    help='Hydra project like nixos/release-16.09')
+    default="nixos/release-17.03",
+    help='Hydra project like nixos/release-17.03')
 def cli(jobset):
     """
     Given a Hydra project, inspect latest evaluation
@@ -74,13 +74,13 @@ def cli(jobset):
     # TODO: dependency failed without propagated builds
     for tr in d('img[alt="Failed"]').parents('tr'):
         a = pq(tr)('a')[1]
-        print "- [ ] [{}]({})".format(a.text, a.get('href'))
+        print("- [ ] [{}]({})".format(a.text, a.get('href')))
 
         sys.stdout.flush()
 
         maintainers = get_maintainers(a.text)
         if maintainers:
-            print "  - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers)))
+            print("  - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers))))
         # TODO: print last three persons that touched this file
         # TODO: pinpoint the diff that broke this build, or maybe it's transient or maybe it never worked?
 
diff --git a/maintainers/scripts/nix-diff.sh b/maintainers/scripts/nix-diff.sh
new file mode 100755
index 00000000000..0c65e29cf43
--- /dev/null
+++ b/maintainers/scripts/nix-diff.sh
@@ -0,0 +1,277 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils gnugrep gnused
+
+################################################################################
+# nix-diff.sh                                                                  #
+################################################################################
+# This script "diffs" Nix profile generations.                                 #
+#                                                                              #
+# Example:                                                                     #
+################################################################################
+# > nix-diff.sh 90 92                                                          #
+# + gnumake-4.2.1                                                              #
+# + gnumake-4.2.1-doc                                                          #
+# - htmldoc-1.8.29                                                             #
+################################################################################
+# The example shows that as of generation 92 and since generation 90,          #
+# gnumake-4.2.1 and gnumake-4.2.1-doc have been installed, while               #
+# htmldoc-1.8.29 has been removed.                                             #
+#                                                                              #
+# The example above shows the default, minimal output mode of this script.     #
+# For more features, run `nix-diff.sh -h` for usage instructions.              #
+################################################################################
+
+usage() {
+    cat <<EOF
+usage: nix-diff.sh [-h | [-p profile | -s] [-q] [-l] [range]]
+-h:         print this message before exiting
+-q:         list the derivations installed in the parent generation
+-l:         diff every available intermediate generation between parent and
+            child
+-p profile: specify the Nix profile to use
+            * defaults to ~/.nix-profile
+-s:         use the system profile
+            * equivalent to: -p /nix/var/nix/profiles/system
+profile:    * should be something like /nix/var/nix/profiles/default, not a
+              generation link like /nix/var/nix/profiles/default-2-link
+range:      the range of generations to diff
+            * the following patterns are allowed, where A, B, and N are positive
+              integers, and G is the currently active generation:
+                A..B => diffs from generation A to generation B
+                ~N   => diffs from the Nth newest generation (older than G) to G
+                A    => diffs from generation A to G
+            * defaults to ~1
+EOF
+}
+
+usage_tip() {
+    echo 'run `nix-diff.sh -h` for usage instructions' >&2
+    exit 1
+}
+
+while getopts :hqlp:s opt; do
+    case $opt in
+        h)
+            usage
+            exit
+            ;;
+        q)
+            opt_query=1
+            ;;
+        l)
+            opt_log=1
+            ;;
+        p)
+            opt_profile=$OPTARG
+            ;;
+        s)
+            opt_profile=/nix/var/nix/profiles/system
+            ;;
+        \?)
+            echo "error: invalid option -$OPTARG" >&2
+            usage_tip
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -n "$opt_profile" ]; then
+    if ! [ -L "$opt_profile" ]; then
+        echo "error: expecting \`$opt_profile\` to be a symbolic link" >&2
+        usage_tip
+    fi
+else
+    opt_profile=$(readlink ~/.nix-profile)
+    if (( $? != 0 )); then
+        echo 'error: unable to dereference `~/.nix-profile`' >&2
+        echo 'specify the profile manually with the `-p` flag' >&2
+        usage_tip
+    fi
+fi
+
+list_gens() {
+    nix-env -p "$opt_profile" --list-generations \
+        | sed -r 's:^\s*::' \
+        | cut -d' ' -f1
+}
+
+current_gen() {
+    nix-env -p "$opt_profile" --list-generations \
+        | grep -E '\(current\)\s*$' \
+        | sed -r 's:^\s*::' \
+        | cut -d' ' -f1
+}
+
+neg_gen() {
+    local i=0 from=$1 n=$2 tmp
+    for gen in $(list_gens | sort -rn); do
+        if ((gen < from)); then
+            tmp=$gen
+            ((i++))
+            ((i == n)) && break
+        fi
+    done
+    if ((i < n)); then
+        echo -n "error: there aren't $n generation(s) older than" >&2
+        echo " generation $from" >&2
+        return 1
+    fi
+    echo $tmp
+}
+
+match() {
+    argv=("$@")
+    for i in $(seq $(($#-1))); do
+        if grep -E "^${argv[$i]}\$" <(echo "$1") >/dev/null; then
+            echo $i
+            return
+        fi
+    done
+    echo 0
+}
+
+case $(match "$1" '' '[0-9]+' '[0-9]+\.\.[0-9]+' '~[0-9]+') in
+    1)
+        diffTo=$(current_gen)
+        diffFrom=$(neg_gen $diffTo 1)
+        (($? == 1)) && usage_tip
+        ;;
+    2)
+        diffFrom=$1
+        diffTo=$(current_gen)
+        ;;
+    3)
+        diffFrom=${1%%.*}
+        diffTo=${1##*.}
+        ;;
+    4)
+        diffTo=$(current_gen)
+        diffFrom=$(neg_gen $diffTo ${1#*~})
+        (($? == 1)) && usage_tip
+        ;;
+    0)
+        echo 'error: invalid invocation' >&2
+        usage_tip
+        ;;
+esac
+
+dirA="${opt_profile}-${diffFrom}-link"
+dirB="${opt_profile}-${diffTo}-link"
+
+declare -a temp_files
+temp_length() {
+    echo -n ${#temp_files[@]}
+}
+temp_make() {
+    temp_files[$(temp_length)]=$(mktemp)
+}
+temp_clean() {
+    rm -f ${temp_files[@]}
+}
+temp_name() {
+    echo -n "${temp_files[$(($(temp_length)-1))]}"
+}
+trap 'temp_clean' EXIT
+
+temp_make
+versA=$(temp_name)
+refs=$(nix-store -q --references "$dirA")
+(( $? != 0 )) && exit 1
+echo "$refs" \
+    | grep -v env-manifest.nix \
+    | sort \
+          > "$versA"
+
+print_tag() {
+    local gen=$1
+    nix-env -p "$opt_profile" --list-generations \
+        | grep -E "^\s*${gen}" \
+        | sed -r 's:^\s*::' \
+        | sed -r 's:\s*$::'
+}
+
+if [ -n "$opt_query" ]; then
+    print_tag $diffFrom
+    cat "$versA" \
+        | sed -r 's:^[^-]+-(.*)$:    \1:'
+
+    print_line=1
+fi
+
+if [ -n "$opt_log" ]; then
+    gens=$(for gen in $(list_gens); do
+               ((diffFrom < gen && gen < diffTo)) && echo $gen
+           done)
+    # Force the $diffTo generation to be included in this list, instead of using
+    # `gen <= diffTo` in the preceding loop, so we encounter an error upon the
+    # event of its nonexistence.
+    gens=$(echo "$gens"
+           echo $diffTo)
+else
+    gens=$diffTo
+fi
+
+temp_make
+add=$(temp_name)
+temp_make
+rem=$(temp_name)
+temp_make
+out=$(temp_name)
+
+for gen in $gens; do
+
+    [ -n "$print_line" ] && echo
+
+    temp_make
+    versB=$(temp_name)
+
+    dirB="${opt_profile}-${gen}-link"
+    refs=$(nix-store -q --references "$dirB")
+    (( $? != 0 )) && exit 1
+    echo "$refs" \
+        | grep -v env-manifest.nix \
+        | sort \
+              > "$versB"
+
+    in=$(comm -3 -1 "$versA" "$versB")
+    sed -r 's:^[^-]*-(.*)$:\1+:' <(echo "$in") \
+        | sort -f \
+               > "$add"
+
+    un=$(comm -3 -2 "$versA" "$versB")
+    sed -r 's:^[^-]*-(.*)$:\1-:' <(echo "$un") \
+        | sort -f \
+               > "$rem"
+
+    cat "$rem" "$add" \
+        | sort -f \
+        | sed -r 's:(.*)-$:- \1:' \
+        | sed -r 's:(.*)\+$:\+ \1:' \
+        | grep -v '^$' \
+              > "$out"
+
+    if [ -n "$opt_query" -o -n "$opt_log" ]; then
+
+        lines=$(wc -l "$out" | cut -d' ' -f1)
+        tag=$(print_tag "$gen")
+        (( $? != 0 )) && exit 1
+        if [ $lines -eq 0 ]; then
+            echo "$tag   (no change)"
+        else
+            echo "$tag"
+        fi
+        cat "$out" \
+            | sed 's:^:    :'
+
+        print_line=1
+
+    else
+        echo "diffing from generation $diffFrom to $diffTo"
+        cat "$out"
+    fi
+
+    versA=$versB
+
+done
+
+exit 0
diff --git a/maintainers/scripts/rebuild-amount.sh b/maintainers/scripts/rebuild-amount.sh
index ebc5dc3b87e..098a8c88cb7 100755
--- a/maintainers/scripts/rebuild-amount.sh
+++ b/maintainers/scripts/rebuild-amount.sh
@@ -1,260 +1,115 @@
-#!/bin/sh
-
-usage () {
-  echo 1>&2 "
-usage:
-  $0
-    [--git commit..commit | --git commit]
-    [--svn rev:rev | --svn rev]
-    [--path path[:path]*]
-    [--help]
-
-This program is used to investigate how any changes inside your nixpkgs
-repository may hurt.  With these kind of information you may choose wisely
-where you should commit your changes.
-
-This program adapts it-self to your versionning system to avoid too much
-effort on your Internet bandwidth.  If you need to check more than one
-commits / revisions, you may use the following commands:
-
-  --git remotes/trunk..master
-  --svn 17670:17677
-
-    Check the differences between each commit separating the first and the
-    last commit.
-
-  --path /etc/nixos/nixpkgs:/tmp/nixpkgs_1:/tmp/nixpkgs_2
-
-    Check the differences between multiple directories containing different
-    versions of nixpkgs.
-
-All these options exist with one commit / revision argument.  Such options
-are used to compare your \$NIXPKGS path with the specified version.
-
-If you omit to mention any other commit / revision, then your \$NIXPKGS path
-is compared with its last update.  This command is useful to test code from
-a dirty repository.
-
-"
-
-  exit 1;
-}
-
-#####################
-# Process Arguments #
-#####################
-
-: ${NIXPKGS=/etc/nixos/nixpkgs/}
-
-vcs=""
-gitCommits=""
-svnRevisions=""
-pathLocations=""
-verbose=false
-
-argfun=""
-for arg; do
-  if test -z "$argfun"; then
-    case $arg in
-      --git) vcs="git"; argfun="set_gitCommits";;
-      --svn) vcs="svn"; argfun="set_svnRevisions";;
-      --path) vcs="path"; argfun="set_pathLocations";;
-      --verbose) verbose=true;;
-      --help) usage;;
-      *) usage;;
-    esac
-  else
-    case $argfun in
-      set_*)
-        var=$(echo $argfun | sed 's,^set_,,')
-        eval $var=$arg
-        ;;
-    esac
-    argfun=""
-  fi
-done
-
-if $verbose; then
-  set -x
-else
-  set +x
+#!/usr/bin/env bash
+set -e
+
+if [ "$#" != 1 ] && [ "$#" != 2 ]; then
+	cat <<-EOF
+	Usage: $0 commit-spec [commit-spec]
+	    You need to be in a git-controlled nixpkgs tree.
+	    The current state of the tree will be used if the second commit is missing.
+	EOF
+	exit 1
 fi
 
-############################
-# Find the repository type #
-############################
+# A slightly hacky way to get the config.
+parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
+			| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
 
-if test -z "$vcs"; then
-  if test -x "$NIXPKGS/.git"; then
-    if git --git-dir="$NIXPKGS/.git" branch > /dev/null 2>&1; then
-      vcs="git"
-      gitCommits=$(git --git-dir="$NIXPKGS/.git" log -n 1 --pretty=format:%H 2> /dev/null)
-    fi
-  elif test -x "$NIXPKGS/.svn"; then
-    cd "$NIXPKGS"
-    if svn info > /dev/null 2>&1; then
-      vcs="svn";
-      svnRevisions=$(svn info | sed -n 's,Revision: ,,p')
-    fi
-    cd -
-  else
-    usage
-  fi
-fi
+echo "Estimating rebuild amount by counting changed Hydra jobs."
 
-###############################
-# Define a storage directory. #
-###############################
+toRemove=()
 
-pkgListDir=""
-exitCode=1
-cleanup(){
-  test -e "$pkgListDir" && rm -rf "$pkgListDir"
-  exit $exitCode;
+cleanup() {
+	rm -rf "${toRemove[@]}"
 }
-
 trap cleanup EXIT SIGINT SIGQUIT ERR
 
-pkgListDir=$(mktemp --tmpdir -d rebuild-amount-XXXXXXXX)
-vcsDir="$pkgListDir/.vcs"
-
-###########################
-# Versionning for Dummies #
-###########################
-
-path_init() {
-  if test "${pathLocations#*:}" = "$pathLocations"; then
-    pathLocations="$NIXPKGS:$pathLocations"
-  fi
-  pathLocations="${pathLocations}:"
+MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
+
+nixexpr() {
+	cat <<-EONIX
+		let
+		  lib = import $1/lib;
+		  hydraJobs = import $1/pkgs/top-level/release.nix
+		    # Compromise: accuracy vs. resources needed for evaluation.
+		    { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
+		  cfg = (import $1 {}).config.rebuild-amount or {};
+
+		  recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
+
+		  # hydraJobs leaves recurseForDerivations as empty attrmaps;
+		  # that would break nix-env and we also need to recurse everywhere.
+		  tweak = lib.mapAttrs
+		    (name: val:
+		      if name == "recurseForDerivations" then true
+		      else if lib.isAttrs val && val.type or null != "derivation"
+		              then recurseIntoAttrs (tweak val)
+		      else val
+		    );
+
+		  # Some of these contain explicit references to platform(s) we want to avoid;
+		  # some even (transitively) depend on ~/.nixpkgs/config.nix (!)
+		  blacklist = [
+		    "tarball" "metrics" "manual"
+		    "darwin-tested" "unstable" "stdenvBootstrapTools"
+		    "moduleSystem" "lib-tests" # these just confuse the output
+		  ];
+		
+		in
+		  tweak (builtins.removeAttrs hydraJobs blacklist)
+	EONIX
 }
 
-path_getNext() {
-  pathLoc="${pathLocations%%:*}"
-  pathLocations="${pathLocations#*:}"
+# Output packages in tree $2 that weren't in $1.
+# Changing the output hash or name is taken as a change.
+# Extra nix-env parameters can be in $3
+newPkgs() {
+	# We use files instead of pipes, as running multiple nix-env processes
+	# could eat too much memory for a standard 4GiB machine.
+	local -a list
+	for i in 1 2; do
+		local l="$($MKTEMP)"
+		list[$i]="$l"
+		toRemove+=("$l")
+
+		local expr="$($MKTEMP)"
+		toRemove+=("$expr")
+		nixexpr "${!i}" > "$expr"
+
+		nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
+			| sort > "${list[$i]}" &
+
+		if [ "$parallel" != "true" ]; then
+			wait
+		fi
+	done
+
+	wait
+	comm -13 "${list[@]}"
 }
 
-path_setPath() {
-  path="$pathLoc"
-}
-
-path_setName() {
-  name=$(echo "$pathLoc" | tr '/' '_')
-}
-
-################
-# Git Commands #
-################
-
-git_init() {
-  git clone "$NIXPKGS/.git" "$vcsDir" > /dev/null 2>&1
-  if echo "gitCommits" | grep -c "\.\." > /dev/null 2>&1; then
-    gitCommits=$(git --git-dir="$vcsDir/.git" log --reverse --pretty=format:%H $gitCommits 2> /dev/null)
-  else
-    pathLocations="$vcsDir:$NIXPKGS"
-    vcs="path"
-    path_init
-  fi
-}
-
-git_getNext() {
-  git --git-dir="$vcsDir/.git" checkout $(echo "$gitCommits" | head -n 1) > /dev/null 2>&1
-  gitCommits=$(echo "$gitCommits" | sed '1 d')
-}
-
-git_setPath() {
-  path="$vcsDir"
-}
-
-git_setName() {
-  name=$(git --git-dir="$vcsDir/.git" log -n 1 --pretty=format:%H  2> /dev/null)
-}
-
-#######################
-# Subversion Commands #
-#######################
-
-svn_init() {
-  cp -r "$NIXPKGS" "$vcsDir" > /dev/null 2>&1
-  if echo "svnRevisions" | grep -c ":" > /dev/null 2>&1; then
-    svnRevisions=$(seq ${svnRevisions%:*} ${svnRevisions#*:})
-  else
-    pathLocations="$vcsDir:$NIXPKGS"
-    vcs="path"
-    path_init
-  fi
-}
-
-svn_getNext() {
-  cd "$vcsDir"
-  svn checkout $(echo "$svnRevisions" | head -n 1) > /dev/null 2>&1
-  cd -
-  svnRevisions=$(echo "$svnRevisions" | sed '1 d')
-}
-
-svn_setPath() {
-  path="$vcsDir"
-}
-
-svn_setName() {
-  name=$(svn info  2> /dev/null | sed -n 's,Revision: ,,p')
-}
-
-####################
-# Logical Commands #
-####################
-
-init    () { ${vcs}_init; }
-getNext () { ${vcs}_getNext; }
-setPath () { ${vcs}_setPath; }
-setName () { ${vcs}_setName; }
-
-
-#####################
-# Check for Rebuild #
-#####################
-
-# Generate the list of all derivations that could be build from a nixpkgs
-# respository.  This list of derivation hashes is compared with previous
-# lists and a brief summary is produced on the output.
-
-compareNames () {
-    nb=$(diff -y --suppress-common-lines --speed-large-files "$pkgListDir/$1.drvs" "$pkgListDir/$2.drvs" 2> /dev/null | wc -l)
-    echo "$1 -> $2: $nb"
-}
-
-echo "Please wait, this may take some minutes ..."
-
-init
-first=""
-oldPrev=""
-
-prev=""
-curr=""
-
-while true; do
-  getNext
-  setPath # set path=...
-  setName # set name=...
-  curr="$name"
-
-  test -z "$curr" && break || true
-
-  nix-instantiate "$path" > "$pkgListDir/$curr.drvs" > /dev/null 2>&1 || true
-
-  if test -n "$prev"; then
-    compareNames "$prev" "$curr"
-  else
-    echo "Number of package to rebuild:"
-    first="$curr"
-  fi
-  oldPrev="$prev"
-  prev="$curr"
+# Prepare nixpkgs trees.
+declare -a tree
+for i in 1 2; do
+	if [ -n "${!i}" ]; then # use the given commit
+		dir="$($MKTEMP -d)"
+		tree[$i]="$dir"
+		toRemove+=("$dir")
+
+		git clone --shared --no-checkout --quiet . "${tree[$i]}"
+		(cd "${tree[$i]}" && git checkout --quiet "${!i}")
+	else #use the current tree
+		tree[$i]="$(pwd)"
+	fi
 done
 
-if test "$first" != "$oldPrev"; then
-  echo "Number of package to rebuild (first -> last):"
-  compareNames "$first" "$curr"
-fi
+newlist="$($MKTEMP)"
+toRemove+=("$newlist")
+# Notes:
+#	- the evaluation is done on x86_64-linux, like on Hydra.
+#	- using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
+newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
+
+# Hacky: keep only the last word of each attribute path and sort.
+sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
+	| sort | uniq -c
 
-exitCode=0
diff --git a/maintainers/scripts/update-python-libraries b/maintainers/scripts/update-python-libraries
new file mode 100755
index 00000000000..7c73510c353
--- /dev/null
+++ b/maintainers/scripts/update-python-libraries
@@ -0,0 +1,243 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])'
+
+"""
+Update a Python package expression by passing in the `.nix` file, or the directory containing it.
+You can pass in multiple files or paths.
+
+You'll likely want to use
+``
+  $ ./update-python-libraries ../../pkgs/development/python-modules/*
+``
+to update all libraries in that folder.
+"""
+
+import argparse
+import logging
+import os
+import re
+import requests
+import toolz
+
+INDEX = "https://pypi.io/pypi"
+"""url of PyPI"""
+
+EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl']
+"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+
+def _get_values(attribute, text):
+    """Match attribute in text and return all matches.
+
+    :returns: List of matches.
+    """
+    regex = '{}\s+=\s+"(.*)";'.format(attribute)
+    regex = re.compile(regex)
+    values = regex.findall(text)
+    return values
+
+def _get_unique_value(attribute, text):
+    """Match attribute in text and return unique match.
+
+    :returns: Single match.
+    """
+    values = _get_values(attribute, text)
+    n = len(values)
+    if n > 1:
+        raise ValueError("found too many values for {}".format(attribute))
+    elif n == 1:
+        return values[0]
+    else:
+        raise ValueError("no value found for {}".format(attribute))
+
+def _get_line_and_value(attribute, text):
+    """Match attribute in text. Return the line and the value of the attribute."""
+    regex = '({}\s+=\s+"(.*)";)'.format(attribute)
+    regex = re.compile(regex)
+    value = regex.findall(text)
+    n = len(value)
+    if n > 1:
+        raise ValueError("found too many values for {}".format(attribute))
+    elif n == 1:
+        return value[0]
+    else:
+        raise ValueError("no value found for {}".format(attribute))
+
+
+def _replace_value(attribute, value, text):
+    """Search and replace value of attribute in text."""
+    old_line, old_value = _get_line_and_value(attribute, text)
+    new_line = old_line.replace(old_value, value)
+    new_text = text.replace(old_line, new_line)
+    return new_text
+
+def _fetch_page(url):
+    r = requests.get(url)
+    if r.status_code == requests.codes.ok:
+        return r.json()
+    else:
+        raise ValueError("request for {} failed".format(url))
+
+def _get_latest_version_pypi(package, extension):
+    """Get latest version and hash from PyPI."""
+    url = "{}/{}/json".format(INDEX, package)
+    json = _fetch_page(url)
+
+    version = json['info']['version']
+    for release in json['releases'][version]:
+        if release['filename'].endswith(extension):
+            # TODO: In case of wheel we need to do further checks!
+            sha256 = release['digests']['sha256']
+            break
+    else:
+        sha256 = None
+    return version, sha256
+
+
+def _get_latest_version_github(package, extension):
+    raise ValueError("updating from GitHub is not yet supported.")
+
+
+FETCHERS = {
+    'fetchFromGitHub'   :   _get_latest_version_github,
+    'fetchPypi'         :   _get_latest_version_pypi,
+    'fetchurl'          :   _get_latest_version_pypi,
+}
+
+
+DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz'
+
+
+FORMATS = {
+    'setuptools'        :   DEFAULT_SETUPTOOLS_EXTENSION,
+    'wheel'             :   'whl'
+}
+
+def _determine_fetcher(text):
+    # Count occurences of fetchers.
+    nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys())
+    if nfetchers == 0:
+        raise ValueError("no fetcher.")
+    elif nfetchers > 1:
+        raise ValueError("multiple fetchers.")
+    else:
+        # Then we check which fetcher to use.
+        for fetcher in FETCHERS.keys():
+            if 'src = {}'.format(fetcher) in text:
+                return fetcher
+
+
+def _determine_extension(text, fetcher):
+    """Determine what extension is used in the expression.
+
+    If we use:
+    - fetchPypi, we check if format is specified.
+    - fetchurl, we determine the extension from the url.
+    - fetchFromGitHub we simply use `.tar.gz`.
+    """
+    if fetcher == 'fetchPypi':
+        try:
+            format = _get_unique_value('format', text)
+        except ValueError as e:
+            format = None   # format was not given
+
+        try:
+            extension = _get_unique_value('extension', text)
+        except ValueError as e:
+            extension = None    # extension was not given
+
+        if extension is None:
+            if format is None:
+                format = 'setuptools'
+            extension = FORMATS[format]
+
+    elif fetcher == 'fetchurl':
+        url = _get_unique_value('url', text)
+        extension = os.path.splitext(url)[1]
+        if 'pypi' not in url:
+            raise ValueError('url does not point to PyPI.')
+
+    elif fetcher == 'fetchFromGitHub':
+        raise ValueError('updating from GitHub is not yet implemented.')
+
+    return extension
+
+
+def _update_package(path):
+
+
+
+    # Read the expression
+    with open(path, 'r') as f:
+        text = f.read()
+
+    # Determine pname.
+    pname = _get_unique_value('pname', text)
+
+    # Determine version.
+    version = _get_unique_value('version', text)
+
+    # First we check how many fetchers are mentioned.
+    fetcher = _determine_fetcher(text)
+
+    extension = _determine_extension(text, fetcher)
+
+    new_version, new_sha256 = _get_latest_version_pypi(pname, extension)
+
+    if new_version == version:
+        logging.info("Path {}: no update available for {}.".format(path, pname))
+        return False
+    if not new_sha256:
+        raise ValueError("no file available for {}.".format(pname))
+
+    text = _replace_value('version', new_version, text)
+    text = _replace_value('sha256', new_sha256, text)
+
+    with open(path, 'w') as f:
+        f.write(text)
+
+        logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
+
+    return True
+
+
+def _update(path):
+
+    # We need to read and modify a Nix expression.
+    if os.path.isdir(path):
+        path = os.path.join(path, 'default.nix')
+
+    # If a default.nix does not exist, we quit.
+    if not os.path.isfile(path):
+        logging.info("Path {}: does not exist.".format(path))
+        return False
+
+    # If file is not a Nix expression, we quit.
+    if not path.endswith(".nix"):
+        logging.info("Path {}: does not end with `.nix`.".format(path))
+        return False
+
+    try:
+        return _update_package(path)
+    except ValueError as e:
+        logging.warning("Path {}: {}".format(path, e))
+        return False
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('package', type=str, nargs='+')
+
+    args = parser.parse_args()
+
+    packages = map(os.path.abspath, args.package)
+
+    count = list(map(_update, packages))
+
+    logging.info("{} package(s) updated".format(sum(count)))
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file