Merge pull request #7501 from wizeman/u/upd-rust

Add support for Rust / Cargo packaging

8 files changed, 328 insertions, 6 deletions

diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh
index 71374d1f238..64eea4ae100 100644
--- a/pkgs/build-support/fetchgit/builder.sh
+++ b/pkgs/build-support/fetchgit/builder.sh
@@ -9,6 +9,7 @@ header "exporting $url (rev $rev) into $out"
 $fetcher --builder --url "$url" --out "$out" --rev "$rev" \
   ${leaveDotGit:+--leave-dotGit} \
   ${deepClone:+--deepClone} \
-  ${fetchSubmodules:+--fetch-submodules}
+  ${fetchSubmodules:+--fetch-submodules} \
+  ${branchName:+--branch-name "$branchName"}
 
 stopNest
diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix
index bb89a8f5532..e5ad7200cec 100644
--- a/pkgs/build-support/fetchgit/default.nix
+++ b/pkgs/build-support/fetchgit/default.nix
@@ -13,6 +13,7 @@
 in
 { url, rev ? "HEAD", md5 ? "", sha256 ? "", leaveDotGit ? deepClone
 , fetchSubmodules ? true, deepClone ? false
+, branchName ? null
 , name ? urlToName url rev
 }:
 
@@ -51,7 +52,7 @@ stdenv.mkDerivation {
   outputHashMode = "recursive";
   outputHash = if sha256 == "" then md5 else sha256;
 
-  inherit url rev leaveDotGit fetchSubmodules deepClone;
+  inherit url rev leaveDotGit fetchSubmodules deepClone branchName;
 
   GIT_SSL_CAINFO = "${cacert}/etc/ca-bundle.crt";
 
diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git
index 486fd3acafb..ceedf313f28 100755
--- a/pkgs/build-support/fetchgit/nix-prefetch-git
+++ b/pkgs/build-support/fetchgit/nix-prefetch-git
@@ -8,6 +8,7 @@ deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
 leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
 fetchSubmodules=
 builder=
+branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
 
 if test -n "$deepClone"; then
     deepClone=true
@@ -31,6 +32,7 @@ for arg; do
             --url) argfun=set_url;;
             --rev) argfun=set_rev;;
             --hash) argfun=set_hashType;;
+            --branch-name) argfun=set_branchName;;
             --deepClone) deepClone=true;;
             --no-deepClone) deepClone=false;;
             --leave-dotGit) leaveDotGit=true;;
@@ -108,7 +110,7 @@ checkout_hash(){
     fi
 
     git fetch ${builder:+--progress} origin || return 1
-    git checkout -b fetchgit $hash || return 1
+    git checkout -b $branchName $hash || return 1
 }
 
 # Fetch only a branch/tag and checkout it.
@@ -131,7 +133,7 @@ checkout_ref(){
     if test -n "$ref"; then
         # --depth option is ignored on http repository.
         git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
-        git checkout -b fetchgit FETCH_HEAD || return 1
+        git checkout -b $branchName FETCH_HEAD || return 1
     else
         return 1
     fi
@@ -222,7 +224,7 @@ make_deterministic_repo(){
         fi
     done
 
-    # Do a full repack. Must run single-threaded, or else we loose determinism.
+    # Do a full repack. Must run single-threaded, or else we lose determinism.
     git config pack.threads 1
     git repack -A -d -f
     rm -f .git/config
@@ -251,7 +253,7 @@ clone_user_rev() {
             fi;;
     esac
 
-    local full_revision=$(cd $dir && (git rev-parse $rev 2> /dev/null || git rev-parse refs/heads/fetchgit) | tail -n1)
+    local full_revision=$(cd $dir && (git rev-parse $rev 2> /dev/null || git rev-parse refs/heads/$branchName) | tail -n1)
     echo "git revision is $full_revision"
     echo "git human-readable version is $(cd $dir && (git describe $full_revision 2> /dev/null || git describe --tags $full_revision 2> /dev/null || echo -- none --))" >&2
     echo "Commit date is $(cd $dir && git show --no-patch --pretty=%ci $full_revision)"
@@ -268,6 +270,10 @@ clone_user_rev() {
     fi
 }
 
+if test -z "$branchName"; then
+    branchName=fetchgit
+fi
+
 if test -n "$builder"; then
     test -n "$out" -a -n "$url" -a -n "$rev" || usage
     mkdir $out
diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix
new file mode 100644
index 00000000000..ddf2fce3cae
--- /dev/null
+++ b/pkgs/build-support/rust/default.nix
@@ -0,0 +1,93 @@
+{ stdenv, cacert, git, rustc, cargo, rustRegistry }:
+{ name, src, depsSha256, buildInputs ? [], cargoUpdateHook ? "", ... } @ args:
+
+let
+  fetchDeps = import ./fetchcargo.nix {
+    inherit stdenv cacert git rustc cargo rustRegistry;
+  };
+
+  cargoDeps = fetchDeps {
+    inherit name src cargoUpdateHook;
+    sha256 = depsSha256;
+  };
+
+  # The following is the directory name cargo creates when the registry index
+  # URL is file:///dev/null
+  #
+  # It's OK to use /dev/null as the URL because by the time we do this, cargo
+  # won't attempt to update the registry anymore, so the URL is more or less
+  # irrelevant
+  registryIndexDirName = "-ba82b75dd6681d6f";
+
+in stdenv.mkDerivation (args // {
+  inherit cargoDeps rustRegistry;
+
+  patchRegistryDeps = ./patch-registry-deps;
+
+  buildInputs = [ git cargo rustc ] ++ buildInputs;
+
+  configurePhase = args.configurePhase or "true";
+
+  postUnpack = ''
+    echo "Using cargo deps from $cargoDeps"
+
+    cp -r "$cargoDeps" deps
+    chmod +w deps -R
+
+    cat <<EOF > deps/config
+    [registry]
+    index = "file:///dev/null"
+    EOF
+
+    echo "Using rust registry from $rustRegistry"
+
+    ln -s "$rustRegistry" "deps/registry/index/${registryIndexDirName}"
+
+    export CARGO_HOME="$(realpath deps)"
+
+    # Retrieved the Cargo.lock file which we saved during the fetch
+    mv deps/Cargo.lock $sourceRoot/
+
+    (
+        cd $sourceRoot
+
+        cargo fetch
+        cargo clean
+    )
+  '' + (args.postUnpack or "");
+
+  prePatch = ''
+    # Patch registry dependencies, using the scripts in $patchRegistryDeps
+    (
+        set -euo pipefail
+
+        cd ../deps/registry/src/*
+
+        for script in $patchRegistryDeps/*; do
+          # Run in a subshell so that directory changes and shell options don't
+          # affect any following commands
+
+          ( . $script)
+        done
+    )
+  '' + (args.prePatch or "");
+
+  buildPhase = args.buildPhase or ''
+    echo "Running cargo build --release"
+    cargo build --release
+  '';
+
+  checkPhase = args.checkPhase or ''
+    echo "Running cargo test"
+    cargo test
+  '';
+
+  doCheck = args.doCheck or true;
+
+  installPhase = args.installPhase or ''
+    mkdir -p $out/bin
+    for f in $(find target/release -maxdepth 1 -type f); do
+      cp $f $out/bin
+    done;
+  '';
+})
diff --git a/pkgs/build-support/rust/fetch-builder.sh b/pkgs/build-support/rust/fetch-builder.sh
new file mode 100644
index 00000000000..faa17e65328
--- /dev/null
+++ b/pkgs/build-support/rust/fetch-builder.sh
@@ -0,0 +1,17 @@
+source $stdenv/setup
+
+# cargo-fetch needs to write to Cargo.lock, even to do nothing. We
+# create a fake checkout with symlinks and and editable Cargo.lock.
+mkdir copy
+cd copy
+for f in $(ls $src); do
+  ln -s $src/"$f" .
+done
+rm Cargo.lock
+cp $src/Cargo.lock .
+chmod +w Cargo.lock
+
+$fetcher . $out
+
+cd ..
+rm -rf copy
diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps
new file mode 100755
index 00000000000..c7799fb1b1f
--- /dev/null
+++ b/pkgs/build-support/rust/fetch-cargo-deps
@@ -0,0 +1,177 @@
+#! /bin/sh
+
+source $stdenv/setup
+
+set -euo pipefail
+
+src=$(realpath $1)
+out=$(realpath $2)
+
+echo "Fetching $src to $out"
+
+mkdir $out
+
+# Configure cargo to fetch from a local copy of the crates.io registry
+
+echo "Using rust registry from $rustRegistry"
+
+cat <<EOF > $out/config
+[registry]
+index = "file://$rustRegistry"
+EOF
+
+export CARGO_HOME=$out
+cd $src
+
+if [[ ! -f Cargo.lock ]]; then
+    echo "ERROR: The Cargo.lock file doesn't exist"
+    echo
+    echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
+    echo "when the registry is updated."
+
+    exit 1
+fi
+
+# We need to do the following string replacement so that 'cargo fetch'
+# doesn't ignore the versions specified in Cargo.lock
+set +u
+substituteInPlace Cargo.lock \
+    --replace "registry+https://github.com/rust-lang/crates.io-index" \
+              "registry+file://$rustRegistry"
+set -u
+
+# Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
+eval "$cargoUpdateHook"
+
+# Do the fetch
+cargo fetch --verbose
+
+# Now that we have fetched everything, let's make the output deterministic
+
+# Cargo uses the following directory structure for fetched data, where
+# $indexHash is a hash of the registry index URL:
+#
+#
+# /config:
+#
+#     Cargo config file. We'll delete this because it's not deterministic,
+#     and instead recreate it just before running 'cargo build'.
+#
+# /registry/cache/$indexHash/:
+#
+#     This is where tarballs of registry package dependencies are kept
+#     We'll need to keep this, but make sure $indexHash is a fixed name.
+#
+# /registry/index/$indexHash/:
+#
+#     A copy of the registry index is kept here. We can delete this, and
+#     instead, just before running 'cargo build', we'll symlink this
+#     directory to our static copy of the registry in the Nix store.
+#
+# /registry/src/$indexHash/{pkgName-pkgVersion}/:
+#
+#     Here cargo keeps extracted sources of the cached tarballs.
+#     We'll just delete this because cargo will re-populate them from the
+#     tarballs.
+#
+# /git/db/{domain-hash}/:
+#
+#     Here cargo keeps the `.git` directories of git dependencies.
+#     We'll need to keep these, but make them deterministic.
+#
+# /git/checkouts/{domain-hash}/{branchName}/:
+#
+#     Here cargo keeps checked-out sources of the git dependencies.
+#     We can delete this, because cargo will re-populate them from the above
+#     `.git` directories.
+#
+# Let's start
+
+# Remove cargo config file, which points to the ever-changing registry
+rm $out/config
+
+# Save the Cargo.lock file into the output, so that we don't have to do another
+# 'cargo update' during the build (which would try to access the network) for
+# any ad-hoc package updates (through $cargoUpdateHook).
+#
+# We need to replace the rustRegistry URL with something deterministic.
+# Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
+
+set +u
+substituteInPlace Cargo.lock \
+    --replace "registry+file://$rustRegistry" \
+              "registry+file:///dev/null"
+set -u
+mv Cargo.lock $out/
+
+# The following is the $indexHash cargo uses for the registry index when
+# its URL is file:///dev/null, which is the registry index URL we use to make
+# sure our output is deterministic.
+
+registryIndexDirName="-ba82b75dd6681d6f"
+mv $out/registry/cache/* $out/registry/cache/$registryIndexDirName
+
+# The registry index changes all the time, so it's not deterministic
+# We'll symlink it before running 'cargo build'
+rm -rf $out/registry/index/*
+
+# Make git DBs deterministic
+# TODO: test with git submodules
+[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
+    cd "$out/git/checkouts/$name"
+    revs=""
+    for branch in *; do
+        cd "$branch"
+        rev="$(git rev-parse HEAD)"
+        revs="$revs $rev"
+        cd ..
+    done
+
+    (
+        # The following code was adapted from nix-prefetch-git
+
+        cd "$out/git/db/$name"
+
+        export GIT_DIR=.
+
+        # Remove all remote branches
+        git branch -r | while read branch; do
+            git branch -rD "$branch" >&2
+        done
+
+        # Remove all tags
+        git tag | while read tag; do
+            git tag -d "$tag" >&2
+        done
+
+        # Remove all local branches
+        branchrefs=()
+        eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
+
+        for branchref in "${branchrefs[@]}"; do
+            git update-ref -d "$branchref" >&2
+        done
+
+        # Create ad-hoc branches for the revs we need
+        echo "$revs" | while read rev; do
+            echo "Creating git branch b_$rev $rev"
+            git branch b_$rev $rev
+        done
+
+        # Remove files that have timestamps or otherwise have non-deterministic
+        # properties.
+        rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
+
+        # Do a full repack. Must run single-threaded, or else we lose determinism.
+        git config pack.threads 1
+        git repack -A -d -f
+        rm -f config
+
+        # Garbage collect unreferenced objects.
+        git gc --prune=all
+    )
+done)
+
+# Remove unneeded outputs
+[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
+[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
diff --git a/pkgs/build-support/rust/fetchcargo.nix b/pkgs/build-support/rust/fetchcargo.nix
new file mode 100644
index 00000000000..7ebd02a485d
--- /dev/null
+++ b/pkgs/build-support/rust/fetchcargo.nix
@@ -0,0 +1,19 @@
+{ stdenv, cacert, git, rustc, cargo, rustRegistry }:
+{ name ? "cargo-deps", src, sha256, cargoUpdateHook ? "" }:
+
+stdenv.mkDerivation {
+  name = "${name}-fetch";
+  buildInputs = [ rustc cargo git ];
+  builder = ./fetch-builder.sh;
+  fetcher = ./fetch-cargo-deps;
+  inherit src rustRegistry cargoUpdateHook;
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "recursive";
+  outputHash = sha256;
+
+  SSL_CERT_FILE = "${cacert}/etc/ca-bundle.crt";
+
+  impureEnvVars = [ "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" ];
+  preferLocalBuild = true;
+}
diff --git a/pkgs/build-support/rust/patch-registry-deps/pkg-config b/pkgs/build-support/rust/patch-registry-deps/pkg-config
new file mode 100644
index 00000000000..2acf489851e
--- /dev/null
+++ b/pkgs/build-support/rust/patch-registry-deps/pkg-config
@@ -0,0 +1,8 @@
+for dir in pkg-config-*; do
+    [ -d "$dir" ] || continue
+
+    echo "Patching pkg-config registry dep"
+
+    substituteInPlace "$dir/src/lib.rs" \
+        --replace '"/usr"' '"/nix/store/"'
+done