Add Nixpkgs support for Dhall

One of the motivations for this change is the following Discourse
discussion:

https://discourse.dhall-lang.org/t/offline-use-of-prelude/137

Many users have requested Dhall support for "offline" packages
that can be fetched/built/installed using ordinary package
management tools (like Nix) instead of using Dhall's HTTP import system.
I will continue to use the term "offline" to mean Dhall package
builds that do not use Dhall's language support for HTTP imports (and
instead use the package manager's support for HTTP requests, such
as `pkgs.fetchFromGitHub`)

The goal of this change is to document what is the idiomatic way to
implement "offline" Dhall builds by implementing Nixpkgs support
for such builds.  That way when other package management tools ask
me how to package Dhall with their tools I can refer them to how it
is done in Nixpkgs.

This change contains a fully "offline" build for the largest Dhall
package in existence, known as "dhall-packages" (not to be confused
with `dhallPackages`, which is our Nix attribute set containing
Dhall packages).

The trick to implementing offline builds in Dhall is to take
advantage of Dhall's support for semantic integrity checks.  If an
HTTP import is protected by an integrity check and a cached build
product matches the integrity check then the HTTP import is never
resolved and the expression is instead fetched from cache.

By "installing" dependencies in a pre-seeded and isolated cache
we can replace remote HTTP imports with dependencies that have
been built and supplied by Nix instead.

The offline nature of the builds are enforced by compiling the
Haskell interpreter with the `-f-with-http` flag, which disables
the interpreter's support for HTTP imports.  If a user forgets
to supply a necessary dependency as a Nix build product then the
build fails informing them that HTTP imports are disabled.

By default, built packages are "binary distributions", containing
just a cache product and a Dhall expression which can be used to
resolve the corresponding cache product.

Users can also optionally enable a "source distribution" of a package
which already includes the equivalent fully-evaluated Dhall code (for
convenience), but this is disabled by default to keep `/nix/store`
utilization as compact as possible.

9 files changed, 315 insertions, 54 deletions

diff --git a/pkgs/development/dhall-modules/Prelude.nix b/pkgs/development/dhall-modules/Prelude.nix
new file mode 100644
index 00000000000..53f6bace41c
--- /dev/null
+++ b/pkgs/development/dhall-modules/Prelude.nix
@@ -0,0 +1,77 @@
+{ buildDhallPackage, fetchFromGitHub, lib }:
+
+let
+  makePrelude =
+    version:
+    lib.makeOverridable
+      ( { rev, sha256, file ? "package.dhall" }:
+          buildDhallPackage {
+            name = "Prelude-${version}";
+
+            code =
+              let
+                src = fetchFromGitHub {
+                  owner = "dhall-lang";
+
+                  repo = "dhall-lang";
+
+                  inherit rev sha256;
+                };
+
+              in
+                "${src}/Prelude/${file}";
+          }
+      );
+
+in
+  lib.mapAttrs makePrelude {
+    # Prelude versions older than 7.0.0 use old-style union literals, which are
+    # no longer supported by the latest version of the standard
+    "7.0.0" = {
+      rev = "f0509b403ace4b8a72ebb5fa9c473b9aeabeaf33";
+
+      sha256 = "00ldlvqfh411vnrnc41zfnlvgfanwfd3l8hdia8kni3r8q9qmd71";
+    };
+
+    "8.0.0" = {
+      rev = "136a3491753fef251b2087031617d1ee1053f285";
+
+      sha256 = "0haxd5dhi5bmg06a0hx1blpivmwrcnndydwagibj3zvch4knyi2q";
+    };
+
+    "9.0.0" = {
+      rev = "6cbf57c946e7e6576babc23a38320e53ecfa6bee";
+
+      sha256 = "1r06fijszyifq5b4j6libwkm06g8693m9n5c4kq61dvzrjfd2gim";
+    };
+
+    "10.0.0" = {
+      rev = "ecbf82785cff406bbd162bbabf3df6f817c805e0";
+
+      sha256 = "0gxkr9649jqpykdzqjc98gkwnjry8wp469037brfghyidwsm021m";
+    };
+
+    "11.0.0" = {
+      rev = "8098184d17c3aecc82674a7b874077a7641be05a";
+
+      sha256 = "0rdvyxq7mvas82wsfzzpk6imzm8ax4q58l522mx0ks69pacpr3yi";
+    };
+
+    "11.1.0" = {
+      rev = "31e90e1996f6c4cb50e03ccb1f3c45beb4bd278c";
+
+      sha256 = "0rdvyxq7mvas82wsfzzpk6imzm8ax4q58l522mx0ks69pacpr3yi";
+    };
+
+    "12.0.0" = {
+      rev = "9f248138f69ee5e22192dc3d0417d5c77b189e04";
+
+      sha256 = "1gbr0376sfamp0ibhcbxz4vaxr6ipv42y42p5wyksfhz3ls9x5ph";
+    };
+
+    "13.0.0" = {
+      rev = "48db9e1ff1f8881fa4310085834fbc19e313ebf0";
+
+      sha256 = "0kg3rzag3irlcldck63rjspls614bc2sbs3zq44h0pzcz9v7z5h9";
+    };
+  }
diff --git a/pkgs/development/dhall-modules/default.nix b/pkgs/development/dhall-modules/default.nix
deleted file mode 100644
index b6632a86f31..00000000000
--- a/pkgs/development/dhall-modules/default.nix
+++ /dev/null
@@ -1,9 +0,0 @@
-{ pkgs }:
-
-# TODO: add into the toplevel fixpoint instead of using rec
-rec {
-
-  prelude = prelude_3_0_0;
-  prelude_3_0_0 = pkgs.callPackage ./prelude/v3.nix {};
-
-}
diff --git a/pkgs/development/dhall-modules/dhall-kubernetes.nix b/pkgs/development/dhall-modules/dhall-kubernetes.nix
new file mode 100644
index 00000000000..5bc0e08c69b
--- /dev/null
+++ b/pkgs/development/dhall-modules/dhall-kubernetes.nix
@@ -0,0 +1,54 @@
+{ buildDhallPackage, fetchFromGitHub, lib }:
+
+let
+  makeDhallKubernetes =
+    version:
+    lib.makeOverridable
+      ( { rev
+        , sha256
+
+          # The version of the Kubernetes OpenAPI spec to use.
+          #
+          # This defaults to the latest supported Kubernetes if left unspecified.
+          #
+          # This is only supported by dhall-kubernetes version 3.0.0 or newer.
+        , kubernetesVersion ? null
+
+        , file ? "package.dhall"
+        }:
+          buildDhallPackage {
+            name = "dhall-kubernetes-${version}";
+
+            code =
+              let
+                src = fetchFromGitHub {
+                  owner = "dhall-lang";
+
+                  repo = "dhall-kubernetes";
+
+                  inherit rev sha256;
+                };
+
+                prefix =
+                  if kubernetesVersion == null then "" else "${kubernetesVersion}/";
+
+              in
+                "${src}/${prefix}${file}";
+          }
+      );
+
+in
+  lib.mapAttrs makeDhallKubernetes {
+    # 2.1.0 was the first version to introduce a top-level `package.dhall` file
+    "2.1.0" = {
+      rev = "bbfec3d8548b605f1c9628f34029ab4a7d928839";
+
+      sha256 = "10zkigj05khiy6w2sqcm5nw7d47r5k52xq8np8q86h0phy798g96";
+    };
+
+    "3.0.0" = {
+      rev = "3c6d09a9409977cdde58a091d76a6d20509ca4b0";
+
+      sha256 = "1r4awh770ghsrwabh5ddy3jpmrbigakk0h32542n1kh71w3cdq1h";
+    };
+  }
diff --git a/pkgs/development/dhall-modules/dhall-packages.nix b/pkgs/development/dhall-modules/dhall-packages.nix
new file mode 100644
index 00000000000..659c582f3c1
--- /dev/null
+++ b/pkgs/development/dhall-modules/dhall-packages.nix
@@ -0,0 +1,71 @@
+{ buildDhallPackage, dhall-kubernetes, fetchFromGitHub, lib, Prelude }:
+
+let
+  makeDhallPackages =
+    version:
+    lib.makeOverridable
+      ( { rev
+        , sha256
+        , dependencies
+        }:
+          buildDhallPackage {
+            name = "dhall-packages-${version}";
+
+            inherit dependencies;
+
+            code =
+              let
+                src = fetchFromGitHub {
+                  owner = "EarnestResearch";
+
+                  repo = "dhall-packages";
+
+                  inherit rev sha256;
+                };
+
+              in
+                "${src}/package.dhall";
+          }
+      );
+
+in
+  lib.mapAttrs makeDhallPackages {
+    "0.11.1" =
+      let
+        k8s_6a47bd = dhall-kubernetes."3.0.0".override {
+          rev = "6a47bd50c4d3984a13570ea62382a3ad4a9919a4";
+
+          sha256 = "1azqs0x2kia3xw93rfk2mdi8izd7gy9aq6qzbip32gin7dncmfhh";
+        };
+
+        k8s_4ad581 = dhall-kubernetes."3.0.0".override {
+          rev = "4ad58156b7fdbbb6da0543d8b314df899feca077";
+
+          sha256 = "12fm70qbhcainxia388svsay2cfg9iksc6mss0nvhgxhpypgp8r0";
+        };
+
+        k8s_fee24c = dhall-kubernetes."3.0.0".override {
+          rev = "fee24c0993ba0b20190e2fdb94e386b7fb67252d";
+
+          sha256 = "11d93z8y0jzrb8dl43gqha9z96nxxqkl7cbxpz8hw8ky9x6ggayk";
+        };
+
+      in
+        { rev = "8d228f578fbc7bb16c04a7c9ac8c6c7d2e13d1f7";
+
+          sha256 = "1v4y1x13lxy6cxf8xqc6sb0mc4mrd4frkxih95v9q2wxw4vkw2h7";
+
+          dependencies = [
+            (k8s_6a47bd.override { kubernetesVersion = "1.14"; })
+            (k8s_6a47bd.override { kubernetesVersion = "1.15"; })
+            (k8s_6a47bd.override { kubernetesVersion = "1.16"; })
+            (k8s_4ad581.override { file = "types.dhall"; })
+            (k8s_fee24c.override { file = "types/io.k8s.api.core.v1.ServiceSpec.dhall"; })
+            (k8s_fee24c.override { file = "types/io.k8s.api.core.v1.PodTemplateSpec.dhall"; })
+            Prelude."12.0.0"
+            (Prelude."12.0.0".override { file = "JSON/package.dhall"; })
+            (Prelude."12.0.0".override { file = "JSON/Type"; })
+            (Prelude."12.0.0".override { file = "Map/Type"; })
+          ];
+        };
+  }
diff --git a/pkgs/development/dhall-modules/prelude/v3.nix b/pkgs/development/dhall-modules/prelude/v3.nix
deleted file mode 100644
index ef673310ceb..00000000000
--- a/pkgs/development/dhall-modules/prelude/v3.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-{ stdenv, lib, fetchFromGitHub }:
-
-stdenv.mkDerivation {
-  name = "dhall-prelude";
-
-  src = fetchFromGitHub {
-    owner = "dhall-lang";
-    repo = "dhall-lang";
-    # Commit where the v3.0.0 prelude folder was merged into dhall-lang
-    # and a LICENSE file has been added.
-    rev = "f6aa9399f1ac831d66c34104abe6856023c5b2df";
-    sha256 = "0kqjgh3y1l3cb3rj381j7c09547g1vh2dsfzpm08y1qajhhf9vgf";
-  };
-
-  phases = [ "unpackPhase" "installPhase" ];
-
-  installPhase = ''
-    cp -r Prelude $out
-  '';
-
-  meta = {
-    license = lib.licenses.bsd3;
-    maintainers = with lib.maintainers; [ Profpatsch ];
-  };
-}
diff --git a/pkgs/development/interpreters/dhall/build-dhall-package.nix b/pkgs/development/interpreters/dhall/build-dhall-package.nix
new file mode 100644
index 00000000000..739dc9b3d52
--- /dev/null
+++ b/pkgs/development/interpreters/dhall/build-dhall-package.nix
@@ -0,0 +1,83 @@
+{ haskell, haskellPackages, lib, lndir, runCommand, writeText }:
+
+{ name
+
+  # Expressions to add to the cache before interpreting the code
+, dependencies ? []
+
+  # A Dhall expression
+  #
+  # Carefully note that the following expression must be devoid of uncached HTTP
+  # imports.  This is because the expression will be evaluated using an
+  # interpreter with HTTP support disabled, so all HTTP imports have to be
+  # protected by an integrity check that can be satisfied via cached
+  # dependencies.
+  #
+  # You can add a dependency to the cache using the preceding `dependencies`
+  # option
+, code
+
+  # `buildDhallPackage` can include both a "source distribution" in
+  # `source.dhall` and a "binary distribution" in `binary.dhall`:
+  #
+  # * `source.dhall` is a dependency-free αβ-normalized Dhall expression
+  #
+  # * `binary.dhall` is an expression of the form: `missing sha256:${HASH}`
+  #
+  #   This expression requires you to install the cache product located at
+  #   `.cache/dhall/1220${HASH}` to successfully resolve
+  #
+  # By default, `buildDhallPackage` only includes "binary.dhall" to conserve
+  # space within the Nix store, but if you set the following `source` option to
+  # `true` then the package will also include `source.dhall`.
+, source ? false
+}:
+
+let
+  # `buildDhallPackage` requires version 1.25.0 or newer of the Haskell
+  # interpreter for Dhall.  Given that the default version is 1.24.0 we choose
+  # the latest available version instead until the default is upgraded.
+  #
+  # HTTP support is disabled in order to force that HTTP dependencies are built
+  # using Nix instead of using Dhall's support for HTTP imports.
+  dhall =
+    haskell.lib.justStaticExecutables
+      (haskell.lib.appendConfigureFlag
+        haskellPackages.dhall_1_29_0
+        "-f-with-http"
+      );
+
+  file = writeText "${name}.dhall" code;
+
+  cache = ".cache";
+
+  cacheDhall = "${cache}/dhall";
+
+  sourceFile = "source.dhall";
+
+in
+  runCommand name { inherit dependencies; } ''
+    set -eu
+
+    mkdir -p ${cacheDhall}
+
+    for dependency in $dependencies; do
+      ${lndir}/bin/lndir -silent $dependency/${cacheDhall} ${cacheDhall}
+    done
+
+    export XDG_CACHE_HOME=$PWD/${cache}
+
+    mkdir -p $out/${cacheDhall}
+
+    ${dhall}/bin/dhall --alpha --file '${file}' > $out/${sourceFile}
+
+    SHA_HASH=$(${dhall}/bin/dhall hash <<< $out/${sourceFile})
+
+    HASH_FILE="''${SHA_HASH/sha256:/1220}"
+
+    ${dhall}/bin/dhall encode --file $out/${sourceFile} > $out/${cacheDhall}/$HASH_FILE
+
+    echo "missing $SHA_HASH" > $out/binary.dhall
+
+    ${lib.optionalString (!source) "rm $out/${sourceFile}"}
+  ''
diff --git a/pkgs/development/interpreters/dhall/default.nix b/pkgs/development/interpreters/dhall/default.nix
deleted file mode 100644
index 8e1df36e412..00000000000
--- a/pkgs/development/interpreters/dhall/default.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ haskell, haskellPackages, stdenvNoCC }:
-
-let
-  static = haskell.lib.justStaticExecutables haskellPackages.dhall;
-
-in static.overrideAttrs (old: {
-  passthru = old.passthru or {} // {
-    prelude = stdenvNoCC.mkDerivation {
-      name = "dhall-prelude";
-      inherit (old) src;
-      phases = [ "unpackPhase" "installPhase" ];
-      installPhase = ''
-        mkdir $out
-        cp -r Prelude/* $out/
-      '';
-    };
-  };
-})
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index cb4016b4971..197639a4cbf 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -9029,7 +9029,7 @@ in
 
   clooj = callPackage ../development/interpreters/clojure/clooj.nix { };
 
-  dhall = callPackage ../development/interpreters/dhall { };
+  dhall = haskell.lib.justStaticExecutables haskellPackages.dhall;
 
   dhall-nix = haskell.lib.justStaticExecutables haskellPackages.dhall-nix;
 
@@ -9039,7 +9039,7 @@ in
 
   dhall-text = haskell.lib.justStaticExecutables haskellPackages.dhall-text;
 
-  dhallPackages = import ../development/dhall-modules { inherit pkgs; };
+  dhallPackages = callPackages ./dhall-packages.nix { };
 
   duktape = callPackage ../development/interpreters/duktape { };
 
diff --git a/pkgs/top-level/dhall-packages.nix b/pkgs/top-level/dhall-packages.nix
new file mode 100644
index 00000000000..6cdd84c2279
--- /dev/null
+++ b/pkgs/top-level/dhall-packages.nix
@@ -0,0 +1,28 @@
+{ lib
+, newScope
+, overrides ? (self: super: {})
+}:
+
+let
+  packages = self:
+    let
+      callPackage = newScope self;
+
+      buildDhallPackage =
+        callPackage ../development/interpreters/dhall/build-dhall-package.nix { };
+
+    in
+      { inherit buildDhallPackage;
+
+        dhall-kubernetes =
+          callPackage ../development/dhall-modules/dhall-kubernetes.nix { };
+
+        dhall-packages =
+          callPackage ../development/dhall-modules/dhall-packages.nix { };
+
+        Prelude =
+          callPackage ../development/dhall-modules/Prelude.nix { };
+      };
+
+in
+  lib.fix' (lib.extends overrides packages)