Merge branch 'master' into staging-next

Comments on conflicts:
- llvm: d6f401e1 vs. 469ecc70 - docs for 6 and 7 say the default is
  to build all targets, so we should be fine
- some pypi hashes: they were equivalent, just base16 vs. base32

6 files changed, 503 insertions, 12 deletions

diff --git a/doc/functions.xml b/doc/functions.xml
index 4193bb49f77..0d6e2770e6e 100644
--- a/doc/functions.xml
+++ b/doc/functions.xml
@@ -11,7 +11,10 @@
  <xi:include href="functions/overrides.xml" />
  <xi:include href="functions/generators.xml" />
  <xi:include href="functions/debug.xml" />
+ <xi:include href="functions/fetchers.xml" />
+ <xi:include href="functions/trivial-builders.xml" />
  <xi:include href="functions/fhs-environments.xml" />
  <xi:include href="functions/shell.xml" />
  <xi:include href="functions/dockertools.xml" />
+ <xi:include href="functions/prefer-remote-fetch.xml" />
 </chapter>
diff --git a/doc/functions/dockertools.xml b/doc/functions/dockertools.xml
index 501f46a967c..ff446cbfffd 100644
--- a/doc/functions/dockertools.xml
+++ b/doc/functions/dockertools.xml
@@ -24,7 +24,7 @@
 
   <para>
    This function is analogous to the <command>docker build</command> command,
-   in that can used to build a Docker-compatible repository tarball containing
+   in that it can be used to build a Docker-compatible repository tarball containing
    a single image with one or multiple layers. As such, the result is suitable
    for being loaded in Docker with <command>docker load</command>.
   </para>
@@ -190,11 +190,11 @@ buildImage {
     By default <function>buildImage</function> will use a static date of one
     second past the UNIX Epoch. This allows <function>buildImage</function> to
     produce binary reproducible images. When listing images with
-    <command>docker list images</command>, the newly created images will be
+    <command>docker images</command>, the newly created images will be
     listed like this:
    </para>
 <screen><![CDATA[
-$ docker image list
+$ docker images
 REPOSITORY   TAG      IMAGE ID       CREATED        SIZE
 hello        latest   08c791c7846e   48 years ago   25.2MB
 ]]></screen>
@@ -217,7 +217,7 @@ pkgs.dockerTools.buildImage {
     and now the Docker CLI will display a reasonable date and sort the images
     as expected:
 <screen><![CDATA[
-$ docker image list
+$ docker images
 REPOSITORY   TAG      IMAGE ID       CREATED              SIZE
 hello        latest   de2bf4786de6   About a minute ago   25.2MB
 ]]></screen>
@@ -402,7 +402,7 @@ pkgs.dockerTools.buildLayeredImage {
 
   <para>
    This function is analogous to the <command>docker pull</command> command, in
-   that can be used to pull a Docker image from a Docker registry. By default
+   that it can be used to pull a Docker image from a Docker registry. By default
    <link xlink:href="https://hub.docker.com/">Docker Hub</link> is used to pull
    images.
   </para>
@@ -484,7 +484,7 @@ sha256:20d9485b25ecfd89204e843a962c1bd70e9cc6858d65d7f5fadc340246e2116b
 
   <para>
    This function is analogous to the <command>docker export</command> command,
-   in that can used to flatten a Docker image that contains multiple layers. It
+   in that it can be used to flatten a Docker image that contains multiple layers. It
    is in fact the result of the merge of all the layers of the image. As such,
    the result is suitable for being imported in Docker with <command>docker
    import</command>.
@@ -557,7 +557,7 @@ buildImage {
 
   <para>
    Creating base files like <literal>/etc/passwd</literal> or
-   <literal>/etc/login.defs</literal> are necessary for shadow-utils to
+   <literal>/etc/login.defs</literal> is necessary for shadow-utils to
    manipulate users and groups.
   </para>
  </section>
diff --git a/doc/functions/fetchers.xml b/doc/functions/fetchers.xml
new file mode 100644
index 00000000000..b3bd2fe0f45
--- /dev/null
+++ b/doc/functions/fetchers.xml
@@ -0,0 +1,206 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="sec-pkgs-fetchers">
+ <title>Fetcher functions</title>
+
+ <para>
+   When using Nix, you will frequently need to download source code
+   and other files from the internet. Nixpkgs comes with a few helper
+   functions that allow you to fetch fixed-output derivations in a
+   structured way.
+ </para>
+
+ <para>
+   The two fetcher primitives are <function>fetchurl</function> and
+   <function>fetchzip</function>. Both of these have two required
+   arguments, a URL and a hash. The hash is typically
+   <literal>sha256</literal>, although many more hash algorithms are
+   supported. Nixpkgs contributors are currently recommended to use
+   <literal>sha256</literal>. This hash will be used by Nix to
+   identify your source. A typical usage of fetchurl is provided
+   below.
+ </para>
+
+ <programlisting><![CDATA[
+{ stdenv, fetchurl }:
+
+stdenv.mkDerivation {
+  name = "hello";
+  src = fetchurl {
+    url = "http://www.example.org/hello.tar.gz";
+    sha256 = "1111111111111111111111111111111111111111111111111111";
+  };
+}
+]]></programlisting>
+
+ <para>
+   The main difference between <function>fetchurl</function> and
+   <function>fetchzip</function> is in how they store the contents.
+   <function>fetchurl</function> will store the unaltered contents of
+   the URL within the Nix store. <function>fetchzip</function> on the
+   other hand will decompress the archive for you, making files and
+   directories directly accessible in the future.
+   <function>fetchzip</function> can only be used with archives.
+   Despite the name, <function>fetchzip</function> is not limited to
+   .zip files and can also be used with any tarball.
+ </para>
+
+ <para>
+   <function>fetchpatch</function> works very similarly to
+   <function>fetchurl</function> with the same arguments expected. It
+   expects patch files as a source and and performs normalization on
+   them before computing the checksum. For example it will remove
+   comments or other unstable parts that are sometimes added by
+   version control systems and can change over time.
+ </para>
+
+ <para>
+   Other fetcher functions allow you to add source code directly from
+   a VCS such as subversion or git. These are mostly straightforward
+   names based on the name of the command used with the VCS system.
+   Because they give you a working repository, they act most like
+   <function>fetchzip</function>.
+ </para>
+
+ <variablelist>
+   <varlistentry>
+     <term>
+      <literal>fetchsvn</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Subversion. Expects <literal>url</literal> to a
+       Subversion directory, <literal>rev</literal>, and
+       <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchgit</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Git. Expects <literal>url</literal> to a Git repo,
+       <literal>rev</literal>, and <literal>sha256</literal>.
+       <literal>rev</literal> in this case can be full the git commit
+       id (SHA1 hash) or a tag name like
+       <literal>refs/tags/v1.0</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchfossil</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Fossil. Expects <literal>url</literal> to a Fossil
+       archive, <literal>rev</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchcvs</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with CVS. Expects <literal>cvsRoot</literal>,
+       <literal>tag</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchhg</literal>
+     </term>
+     <listitem>
+      <para>
+       Used with Mercurial. Expects <literal>url</literal>,
+       <literal>rev</literal>, and <literal>sha256</literal>.
+      </para>
+     </listitem>
+   </varlistentry>
+ </variablelist>
+
+ <para>
+   A number of fetcher functions wrap part of
+   <function>fetchurl</function> and <function>fetchzip</function>.
+   They are mainly convenience functions intended for commonly used
+   destinations of source code in Nixpkgs. These wrapper fetchers are
+   listed below.
+ </para>
+
+ <variablelist>
+   <varlistentry>
+     <term>
+      <literal>fetchFromGitHub</literal>
+     </term>
+     <listitem>
+      <para>
+        <function>fetchFromGitHub</function> expects four arguments.
+        <literal>owner</literal> is a string corresponding to the
+        GitHub user or organization that controls this repository.
+        <literal>repo</literal> corresponds to the name of the
+        software repository. These are located at the top of every
+        GitHub HTML page as
+        <literal>owner</literal>/<literal>repo</literal>.
+        <literal>rev</literal> corresponds to the Git commit hash or
+        tag (e.g <literal>v1.0</literal>) that will be downloaded from
+        Git. Finally, <literal>sha256</literal> corresponds to the
+        hash of the extracted directory. Again, other hash algorithms
+        are also available but <literal>sha256</literal> is currently
+        preferred.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromGitLab</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with GitLab repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromBitbucket</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with BitBucket repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromSavannah</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with Savannah repositories. The arguments expected
+       are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+   <varlistentry>
+     <term>
+      <literal>fetchFromRepoOrCz</literal>
+     </term>
+     <listitem>
+      <para>
+       This is used with repo.or.cz repositories. The arguments
+       expected are very similar to fetchFromGitHub above.
+      </para>
+     </listitem>
+   </varlistentry>
+ </variablelist>
+
+
+</section>
diff --git a/doc/functions/prefer-remote-fetch.xml b/doc/functions/prefer-remote-fetch.xml
new file mode 100644
index 00000000000..85f08f4eae1
--- /dev/null
+++ b/doc/functions/prefer-remote-fetch.xml
@@ -0,0 +1,27 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/xinclude"
+         xml:id="sec-prefer-remote-fetch">
+ <title>prefer-remote-fetch overlay</title>
+
+ <para>
+  <function>prefer-remote-fetch</function> is an overlay that download sources
+  on remote builder. This is useful when the evaluating machine has a slow
+  upload while the builder can fetch faster directly from the source.
+  To use it, put the following snippet as a new overlay:
+  <programlisting>
+    self: super:
+      (super.prefer-remote-fetch self super)
+  </programlisting>
+
+  A full configuration example for that sets the overlay up for your own account,
+  could look like this
+
+  <programlisting>
+    $ mkdir ~/.config/nixpkgs/overlays/
+    $ cat &gt; ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix &lt;&lt;EOF
+      self: super: super.prefer-remote-fetch self super
+    EOF
+  </programlisting>
+ </para>
+</section>
diff --git a/doc/functions/trivial-builders.xml b/doc/functions/trivial-builders.xml
new file mode 100644
index 00000000000..92a07aedb5b
--- /dev/null
+++ b/doc/functions/trivial-builders.xml
@@ -0,0 +1,124 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="sec-trivial-builders">
+ <title>Trivial builders</title>
+
+ <para>
+   Nixpkgs provides a couple of functions that help with building
+   derivations. The most important one,
+   <function>stdenv.mkDerivation</function>, has already been
+   documented above. The following functions wrap
+   <function>stdenv.mkDerivation</function>, making it easier to use
+   in certain cases.
+ </para>
+
+ <variablelist>
+  <varlistentry>
+   <term>
+    <literal>runCommand</literal>
+   </term>
+   <listitem>
+     <para>
+       This takes three arguments, <literal>name</literal>,
+       <literal>env</literal>, and <literal>buildCommand</literal>.
+       <literal>name</literal> is just the name that Nix will append
+       to the store path in the same way that
+       <literal>stdenv.mkDerivation</literal> uses its
+       <literal>name</literal> attribute. <literal>env</literal> is an
+       attribute set specifying environment variables that will be set
+       for this derivation. These attributes are then passed to the
+       wrapped <literal>stdenv.mkDerivation</literal>.
+       <literal>buildCommand</literal> specifies the commands that
+       will be run to create this derivation. Note that you will need
+       to create <literal>$out</literal> for Nix to register the
+       command as successful.
+     </para>
+     <para>
+       An example of using <literal>runCommand</literal> is provided
+       below.
+     </para>
+     <programlisting>
+       (import &lt;nixpkgs&gt; {}).runCommand "my-example" {} ''
+         echo My example command is running
+
+         mkdir $out
+
+         echo I can write data to the Nix store > $out/message
+
+         echo I can also run basic commands like:
+
+         echo ls
+         ls
+
+         echo whoami
+         whoami
+
+         echo date
+         date
+       ''
+     </programlisting>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>runCommandCC</literal>
+   </term>
+   <listitem>
+     <para>
+       This works just like <literal>runCommand</literal>. The only
+       difference is that it also provides a C compiler in
+       <literal>buildCommand</literal>’s environment. To minimize your
+       dependencies, you should only use this if you are sure you will
+       need a C compiler as part of running your command.
+    </para>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>writeTextFile</literal>, <literal>writeText</literal>,
+    <literal>writeTextDir</literal>, <literal>writeScript</literal>,
+    <literal>writeScriptBin</literal>
+   </term>
+   <listitem>
+     <para>
+       These functions write <literal>text</literal> to the Nix store.
+       This is useful for creating scripts from Nix expressions.
+       <literal>writeTextFile</literal> takes an attribute set and
+       expects two arguments, <literal>name</literal> and
+       <literal>text</literal>. <literal>name</literal> corresponds to
+       the name used in the Nix store path. <literal>text</literal>
+       will be the contents of the file. You can also set
+       <literal>executable</literal> to true to make this file have
+       the executable bit set.
+     </para>
+     <para>
+       Many more commands wrap <literal>writeTextFile</literal>
+       including <literal>writeText</literal>,
+       <literal>writeTextDir</literal>,
+       <literal>writeScript</literal>, and
+       <literal>writeScriptBin</literal>. These are convenience
+       functions over <literal>writeTextFile</literal>.
+     </para>
+   </listitem>
+  </varlistentry>
+  <varlistentry>
+   <term>
+    <literal>symlinkJoin</literal>
+   </term>
+   <listitem>
+    <para>
+     This can be used to put many derivations into the same directory
+     structure. It works by creating a new derivation and adding
+     symlinks to each of the paths listed. It expects two arguments,
+     <literal>name</literal>, and <literal>paths</literal>.
+     <literal>name</literal> is the name used in the Nix store path
+     for the created derivation. <literal>paths</literal> is a list of
+     paths that will be symlinked. These paths can be to Nix store
+     derivations or any other subdirectory contained within.
+    </para>
+   </listitem>
+  </varlistentry>
+ </variablelist>
+
+</section>
diff --git a/doc/stdenv.xml b/doc/stdenv.xml
index 9951b6bf13c..81bd4556193 100644
--- a/doc/stdenv.xml
+++ b/doc/stdenv.xml
@@ -1279,7 +1279,9 @@ makeFlags = [ "PREFIX=$(out)" ];
        <command>make</command>. You must use this instead of
        <varname>makeFlags</varname> if the arguments contain spaces, e.g.
 <programlisting>
-makeFlagsArray=(CFLAGS="-O0 -g" LDFLAGS="-lfoo -lbar")
+preBuild = ''
+  makeFlagsArray+=(CFLAGS="-O0 -g" LDFLAGS="-lfoo -lbar")
+'';
 </programlisting>
        Note that shell arrays cannot be passed through environment variables,
        so you cannot set <varname>makeFlagsArray</varname> in a derivation
@@ -2202,10 +2204,130 @@ addEnvHooks "$hostOffset" myBashFunction
   </para>
 
   <para>
-   Here are some packages that provide a setup hook. Since the mechanism is
-   modular, this probably isn't an exhaustive list. Then again, since the
-   mechanism is only to be used as a last resort, it might be.
-   <variablelist>
+    First, let’s cover some setup hooks that are part of Nixpkgs
+    default stdenv. This means that they are run for every package
+    built using <function>stdenv.mkDerivation</function>. Some of
+    these are platform specific, so they may run on Linux but not
+    Darwin or vice-versa.
+    <variablelist>
+    <varlistentry>
+     <term>
+      <literal>move-docs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any installed documentation to the
+       <literal>/share</literal> subdirectory directory. This includes
+       the man, doc and info directories. This is needed for legacy
+       programs that do not know how to use the
+       <literal>share</literal> subdirectory.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>compress-man-pages.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook compresses any man pages that have been
+       installed. The compression is done using the gzip program. This
+       helps to reduce the installed size of packages.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>strip.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This runs the strip command on installed binaries and
+       libraries. This removes unnecessary information like debug
+       symbols when they are not needed. This also helps to reduce the
+       installed size of packages.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>patch-shebangs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook patches installed scripts to use the full path
+       to the shebang interpreter. A shebang interpreter is the first
+       commented line of a script telling the operating system which
+       program will run the script (e.g <literal>#!/bin/bash</literal>). In
+       Nix, we want an exact path to that interpreter to be used. This
+       often replaces <literal>/bin/sh</literal> with a path in the
+       Nix store.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>audit-tmpdir.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This verifies that no references are left from the install
+       binaries to the directory used to build those binaries. This
+       ensures that the binaries do not need things outside the Nix
+       store. This is currently supported in Linux only.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>multiple-outputs.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook adds configure flags that tell packages to
+       install files into any one of the proper outputs listed in
+       <literal>outputs</literal>. This behavior can be turned off by setting
+       <literal>setOutputFlags</literal> to false in the derivation
+       environment. See <xref linkend="chap-multiple-output"/> for
+       more information.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>move-sbin.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any binaries installed in the sbin
+       subdirectory into bin. In addition, a link is provided from
+       sbin to bin for compatibility.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>move-lib64.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This setup hook moves any libraries installed in the lib64
+       subdirectory into lib. In addition, a link is provided from
+       lib64 to lib for compatibility.
+       </para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>
+      <literal>set-source-date-epoch-to-latest.sh</literal>
+     </term>
+     <listitem>
+       <para>
+       This sets <literal>SOURCE_DATE_EPOCH</literal> to the
+       modification time of the most recent file.
+       </para>
+     </listitem>
+    </varlistentry>
     <varlistentry>
      <term>
       Bintools Wrapper
@@ -2312,6 +2434,15 @@ addEnvHooks "$hostOffset" myBashFunction
       </para>
      </listitem>
     </varlistentry>
+    </variablelist>
+  </para>
+
+  <para>
+   Here are some more packages that provide a setup hook. Since the
+   list of hooks is extensible, this is not an exhaustive list the
+   mechanism is only to be used as a last resort, it might cover most
+   uses.
+   <variablelist>
     <varlistentry>
      <term>
       Perl