rust: fix cross-compilation

6 files changed, 196 insertions, 126 deletions

diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix
index c69832a89b9..7130dde388b 100644
--- a/pkgs/build-support/rust/default.nix
+++ b/pkgs/build-support/rust/default.nix
@@ -1,4 +1,4 @@
-{ stdenv, cacert, git, cargo, rustc, cargo-vendor, fetchcargo, python3 }:
+{ stdenv, cacert, git, cargo, rustc, cargo-vendor, fetchcargo, python3, buildPackages }:
 
 { name ? "${args.pname}-${args.version}"
 , cargoSha256 ? "unset"
@@ -9,6 +9,7 @@
 , sourceRoot ? null
 , logLevel ? ""
 , buildInputs ? []
+, nativeBuildInputs ? []
 , cargoUpdateHook ? ""
 , cargoDepsHook ? ""
 , cargoBuildFlags ? []
@@ -37,21 +38,22 @@ let
       cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
     '';
 
+  ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
+  cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++";
+  ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
+  cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
+  releaseDir = "target/${stdenv.hostPlatform.config}/release";
+
 in stdenv.mkDerivation (args // {
   inherit cargoDeps;
 
   patchRegistryDeps = ./patch-registry-deps;
 
-  buildInputs = [ cacert git cargo rustc ] ++ buildInputs;
+  nativeBuildInputs = [ cargo rustc git cacert ] ++ nativeBuildInputs;
+  inherit buildInputs;
 
   patches = cargoPatches ++ patches;
 
-  configurePhase = args.configurePhase or ''
-    runHook preConfigure
-    # noop
-    runHook postConfigure
-  '';
-
   postUnpack = ''
     eval "$cargoDepsHook"
 
@@ -63,17 +65,40 @@ in stdenv.mkDerivation (args // {
       config=${./fetchcargo-default-config.toml};
     fi;
     substitute $config .cargo/config \
-    --subst-var-by vendor "$(pwd)/$cargoDepsCopy"
+      --subst-var-by vendor "$(pwd)/$cargoDepsCopy"
 
     unset cargoDepsCopy
 
     export RUST_LOG=${logLevel}
   '' + (args.postUnpack or "");
 
+  configurePhase = args.configurePhase or ''
+    runHook preConfigure
+    mkdir -p .cargo
+    cat >> .cargo/config <<'EOF'
+    [target."${stdenv.buildPlatform.config}"]
+    "linker" = "${ccForBuild}"
+    ${stdenv.lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
+    [target."${stdenv.hostPlatform.config}"]
+    "linker" = "${ccForHost}"
+    ''}
+    EOF
+    cat .cargo/config
+    runHook postConfigure
+  '';
+
   buildPhase = with builtins; args.buildPhase or ''
     runHook preBuild
-    echo "Running cargo build --release ${concatStringsSep " " cargoBuildFlags}"
-    cargo build --release --frozen ${concatStringsSep " " cargoBuildFlags}
+    echo "Running cargo build --target ${stdenv.hostPlatform.config} --release ${concatStringsSep " " cargoBuildFlags}"
+    env \
+      "CC_${stdenv.buildPlatform.config}"="${ccForBuild}" \
+      "CXX_${stdenv.buildPlatform.config}"="${cxxForBuild}" \
+      "CC_${stdenv.hostPlatform.config}"="${ccForHost}" \
+      "CXX_${stdenv.hostPlatform.config}"="${cxxForHost}" \
+      cargo build \
+        --release \
+        --target ${stdenv.hostPlatform.config} \
+        --frozen ${concatStringsSep " " cargoBuildFlags}
     runHook postBuild
   '';
 
@@ -86,11 +111,21 @@ in stdenv.mkDerivation (args // {
 
   doCheck = args.doCheck or true;
 
+  inherit releaseDir;
+
   installPhase = args.installPhase or ''
     runHook preInstall
     mkdir -p $out/bin $out/lib
-    find target/release -maxdepth 1 -type f -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \) -print0 | xargs -r -0 cp -t $out/bin
-    find target/release -maxdepth 1 -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" -print0 | xargs -r -0 cp -t $out/lib
+
+    find $releaseDir \
+      -maxdepth 1 \
+      -type f \
+      -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \) \
+      -print0 | xargs -r -0 cp -t $out/bin
+    find $releaseDir \
+      -maxdepth 1 \
+      -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \
+      -print0 | xargs -r -0 cp -t $out/lib
     rmdir --ignore-fail-on-non-empty $out/lib $out/bin
     runHook postInstall
   '';
diff --git a/pkgs/build-support/rust/make-rust-platform.nix b/pkgs/build-support/rust/make-rust-platform.nix
deleted file mode 100644
index afbc56865ff..00000000000
--- a/pkgs/build-support/rust/make-rust-platform.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ callPackage }:
-{ rustc, cargo, ... }: {
-  rust = {
-    inherit rustc cargo;
-  };
-
-  buildRustPackage = callPackage ./default.nix {
-    inherit rustc cargo;
-
-    fetchcargo = callPackage ./fetchcargo.nix {
-      inherit cargo;
-    };
-  };
-
-  rustcSrc = callPackage ../../development/compilers/rust/rust-src.nix {
-    inherit rustc;
-  };
-}
diff --git a/pkgs/development/compilers/rust/cargo.nix b/pkgs/development/compilers/rust/cargo.nix
index 65aa4ea9292..9f576042787 100644
--- a/pkgs/development/compilers/rust/cargo.nix
+++ b/pkgs/development/compilers/rust/cargo.nix
@@ -1,13 +1,11 @@
 { stdenv, file, curl, pkgconfig, python, openssl, cmake, zlib
 , makeWrapper, libiconv, cacert, rustPlatform, rustc, libgit2
 , CoreFoundation, Security
-, version
-, patches ? []
-, src }:
+}:
 
 rustPlatform.buildRustPackage rec {
-  name = "cargo-${version}";
-  inherit version src patches;
+  name = "cargo-${rustc.version}";
+  inherit (rustc) version src;
 
   # the rust source tarball already has all the dependencies vendored, no need to fetch them again
   cargoVendorDir = "vendor";
@@ -19,8 +17,8 @@ rustPlatform.buildRustPackage rec {
   # changes hash of vendor directory otherwise
   dontUpdateAutotoolsGnuConfigScripts = true;
 
-  nativeBuildInputs = [ pkgconfig ];
-  buildInputs = [ cacert file curl python openssl cmake zlib makeWrapper libgit2 ]
+  nativeBuildInputs = [ pkgconfig cmake makeWrapper ];
+  buildInputs = [ cacert file curl python openssl zlib libgit2 ]
     ++ stdenv.lib.optionals stdenv.isDarwin [ CoreFoundation Security libiconv ];
 
   LIBGIT2_SYS_USE_PKG_CONFIG = 1;
diff --git a/pkgs/development/compilers/rust/default.nix b/pkgs/development/compilers/rust/default.nix
index 9640cd9b577..3a6ce5821d9 100644
--- a/pkgs/development/compilers/rust/default.nix
+++ b/pkgs/development/compilers/rust/default.nix
@@ -1,47 +1,67 @@
-{ stdenv, callPackage, recurseIntoAttrs, makeRustPlatform, llvm, fetchurl
+{ stdenv, lib, overrideCC
+, buildPackages
+, newScope, callPackage
 , CoreFoundation, Security
-, targets ? []
-, targetToolchains ? []
-, targetPatches ? []
-}:
+, gcc6
+}: rec {
+  makeRustPlatform = { rustc, cargo, ... }: {
+    rust = {
+      inherit rustc cargo;
+    };
 
-let
-  rustPlatform = recurseIntoAttrs (makeRustPlatform (callPackage ./bootstrap.nix {}));
-  version = "1.32.0";
-  cargoVersion = "1.32.0";
-  src = fetchurl {
-    url = "https://static.rust-lang.org/dist/rustc-${version}-src.tar.gz";
-    sha256 = "0ji2l9xv53y27xy72qagggvq47gayr5lcv2jwvmfirx029vlqnac";
-  };
-in rec {
-  rustc = callPackage ./rustc.nix {
-    inherit stdenv llvm targets targetPatches targetToolchains rustPlatform version src;
-
-    patches = [
-      ./patches/net-tcp-disable-tests.patch
-
-      # Re-evaluate if this we need to disable this one
-      #./patches/stdsimd-disable-doctest.patch
-    ];
-
-    withBundledLLVM = false;
-
-    configureFlags = [ "--release-channel=stable" ];
+    buildRustPackage = callPackage ../../../build-support/rust {
+      inherit rustc cargo;
 
-    # 1. Upstream is not running tests on aarch64:
-    # see https://github.com/rust-lang/rust/issues/49807#issuecomment-380860567
-    # So we do the same.
-    # 2. Tests run out of memory for i686
-    #doCheck = !stdenv.isAarch64 && !stdenv.isi686;
+      fetchcargo = buildPackages.callPackage ../../../build-support/rust/fetchcargo.nix {
+        inherit cargo;
+      };
+    };
 
-    # Disabled for now; see https://github.com/NixOS/nixpkgs/pull/42348#issuecomment-402115598.
-    doCheck = false;
+    rustcSrc = callPackage ./rust-src.nix {
+      inherit rustc;
+    };
   };
 
-  cargo = callPackage ./cargo.nix rec {
-    version = cargoVersion;
-    inherit src stdenv CoreFoundation Security;
-    inherit rustc; # the rustc that will be wrapped by cargo
-    inherit rustPlatform; # used to build cargo
+  # This just contains tools for now. But it would conceivably contain
+  # libraries too, say if we picked some default/recommended versions from
+  # `cratesIO` to build by Hydra and/or try to prefer/bias in Cargo.lock for
+  # all vendored Carnix-generated nix.
+  #
+  # In the end game, rustc, the rust standard library (`core`, `std`, etc.),
+  # and cargo would themselves be built with `buildRustCreate` like
+  # everything else. Tools and `build.rs` and procedural macro dependencies
+  # would be taken from `buildRustPackages` (and `bootstrapRustPackages` for
+  # anything provided prebuilt or their build-time dependencies to break
+  # cycles / purify builds). In this way, nixpkgs would be in control of all
+  # bootstrapping.
+  packages = {
+    prebuilt = callPackage ./bootstrap.nix {};
+    stable = lib.makeScope newScope (self: let
+      # Like `buildRustPackages`, but may also contain prebuilt binaries to
+      # break cycle. Just like `bootstrapTools` for nixpkgs as a whole,
+      # nothing in the final package set should refer to this.
+      bootstrapRustPackages = self.buildRustPackages.overrideScope' (_: _:
+        lib.optionalAttrs (stdenv.buildPlatform == stdenv.hostPlatform)
+          buildPackages.rust.packages.prebuilt);
+      bootRustPlatform = makeRustPlatform bootstrapRustPackages;
+    in {
+      # Packages suitable for build-time, e.g. `build.rs`-type stuff.
+      buildRustPackages = buildPackages.rust.packages.stable;
+      # Analogous to stdenv
+      rustPlatform = makeRustPlatform self.buildRustPackages;
+      rustc = self.callPackage ./rustc.nix ({
+        # Use boot package set to break cycle
+        rustPlatform = bootRustPlatform;
+      } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
+        stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
+      });
+      cargo = self.callPackage ./cargo.nix ({
+        # Use boot package set to break cycle
+        rustPlatform = bootRustPlatform;
+        inherit CoreFoundation Security;
+      } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
+        stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
+      });
+    });
   };
 }
diff --git a/pkgs/development/compilers/rust/rustc.nix b/pkgs/development/compilers/rust/rustc.nix
index 8cddb9ea228..631c20d16c6 100644
--- a/pkgs/development/compilers/rust/rustc.nix
+++ b/pkgs/development/compilers/rust/rustc.nix
@@ -1,33 +1,28 @@
-{ stdenv, targetPackages, removeReferencesTo
+{ stdenv, removeReferencesTo, pkgsBuildBuild, pkgsBuildHost, pkgsBuildTarget
 , fetchurl, fetchgit, fetchzip, file, python2, tzdata, ps
-, llvm, ncurses, darwin, rustPlatform, git, cmake, curl
+, llvm_7, ncurses, darwin, git, cmake, curl, rustPlatform
 , which, libffi, gdb
-, version
 , withBundledLLVM ? false
-, src
-, configureFlags ? []
-, patches
-, targets
-, targetPatches
-, targetToolchains
-, doCheck ? true
-, broken ? false
 }:
 
 let
   inherit (stdenv.lib) optional optionalString;
   inherit (darwin.apple_sdk.frameworks) Security;
 
-  llvmShared = llvm.override { enableSharedLibraries = true; };
+  llvmSharedForBuild = pkgsBuildBuild.llvm_7.override { enableSharedLibraries = true; };
+  llvmSharedForHost = pkgsBuildHost.llvm_7.override { enableSharedLibraries = true; };
+  llvmSharedForTarget = pkgsBuildTarget.llvm_7.override { enableSharedLibraries = true; };
 
-  target = builtins.replaceStrings [" "] [","] (builtins.toString targets);
-in
+  # For use at runtime
+  llvmShared = llvm_7.override { enableSharedLibraries = true; };
+in stdenv.mkDerivation rec {
+  pname = "rustc";
+  version = "1.32.0";
 
-stdenv.mkDerivation {
-  name = "rustc-${version}";
-  inherit version;
-
-  inherit src;
+  src = fetchurl {
+    url = "https://static.rust-lang.org/dist/rustc-${version}-src.tar.gz";
+    sha256 = "0ji2l9xv53y27xy72qagggvq47gayr5lcv2jwvmfirx029vlqnac";
+  };
 
   __darwinAllowLocalNetworking = true;
 
@@ -40,11 +35,12 @@ stdenv.mkDerivation {
   # See https://github.com/NixOS/nixpkgs/pull/34227
   stripDebugList = if stdenv.isDarwin then [ "bin" ] else null;
 
+
   NIX_LDFLAGS =
        # when linking stage1 libstd: cc: undefined reference to `__cxa_begin_catch'
        optional (stdenv.isLinux && !withBundledLLVM) "--push-state --as-needed -lstdc++ --pop-state"
     ++ optional (stdenv.isDarwin && !withBundledLLVM) "-lc++"
-    ++ optional stdenv.isDarwin "-rpath ${llvmShared}/lib";
+    ++ optional stdenv.isDarwin "-rpath ${llvmSharedForHost}/lib";
 
   # Enable nightly features in stable compiles (used for
   # bootstrapping, see https://github.com/rust-lang/rust/pull/37265).
@@ -57,27 +53,67 @@ stdenv.mkDerivation {
 
   # We need rust to build rust. If we don't provide it, configure will try to download it.
   # Reference: https://github.com/rust-lang/rust/blob/master/src/bootstrap/configure.py
-  configureFlags = configureFlags
-                ++ [ "--enable-local-rust" "--local-rust-root=${rustPlatform.rust.rustc}" "--enable-rpath"
-                     "--enable-vendor"
-                     "--default-linker=${targetPackages.stdenv.cc}/bin/cc" ]
-                ++ optional (!withBundledLLVM) [ "--enable-llvm-link-shared" "--llvm-root=${llvmShared}" ]
-                ++ optional (targets != []) "--target=${target}";
+  configureFlags = let
+    setBuild  = "--set=target.${stdenv.buildPlatform.config}";
+    setHost   = "--set=target.${stdenv.hostPlatform.config}";
+    setTarget = "--set=target.${stdenv.targetPlatform.config}";
+    ccForBuild  = "${pkgsBuildBuild.targetPackages.stdenv.cc}/bin/${pkgsBuildBuild.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForBuild = "${pkgsBuildBuild.targetPackages.stdenv.cc}/bin/${pkgsBuildBuild.targetPackages.stdenv.cc.targetPrefix}c++";
+    ccForHost  = "${pkgsBuildHost.targetPackages.stdenv.cc}/bin/${pkgsBuildHost.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForHost = "${pkgsBuildHost.targetPackages.stdenv.cc}/bin/${pkgsBuildHost.targetPackages.stdenv.cc.targetPrefix}c++";
+    ccForTarget  = "${pkgsBuildTarget.targetPackages.stdenv.cc}/bin/${pkgsBuildTarget.targetPackages.stdenv.cc.targetPrefix}cc";
+    cxxForTarget = "${pkgsBuildTarget.targetPackages.stdenv.cc}/bin/${pkgsBuildTarget.targetPackages.stdenv.cc.targetPrefix}c++";
+  in [
+    "--release-channel=stable"
+    "--set=build.rustc=${rustPlatform.rust.rustc}/bin/rustc"
+    "--set=build.cargo=${rustPlatform.rust.cargo}/bin/cargo"
+    "--enable-rpath"
+    "--enable-vendor"
+    "--build=${stdenv.buildPlatform.config}"
+    "--host=${stdenv.hostPlatform.config}"
+    "--target=${stdenv.targetPlatform.config}"
+
+    "${setBuild}.cc=${ccForBuild}"
+    "${setHost}.cc=${ccForHost}"
+    "${setTarget}.cc=${ccForTarget}"
+
+    "${setBuild}.linker=${ccForBuild}"
+    "${setHost}.linker=${ccForHost}"
+    "${setTarget}.linker=${ccForTarget}"
+
+    "${setBuild}.cxx=${cxxForBuild}"
+    "${setHost}.cxx=${cxxForHost}"
+    "${setTarget}.cxx=${cxxForTarget}"
+  ] ++ optional (!withBundledLLVM) [
+    "--enable-llvm-link-shared"
+    "${setBuild}.llvm-config=${llvmSharedForBuild}/bin/llvm-config"
+    "${setHost}.llvm-config=${llvmSharedForHost}/bin/llvm-config"
+    "${setTarget}.llvm-config=${llvmSharedForTarget}/bin/llvm-config"
+  ];
 
   # The bootstrap.py will generated a Makefile that then executes the build.
   # The BOOTSTRAP_ARGS used by this Makefile must include all flags to pass
   # to the bootstrap builder.
   postConfigure = ''
-    substituteInPlace Makefile --replace 'BOOTSTRAP_ARGS :=' 'BOOTSTRAP_ARGS := --jobs $(NIX_BUILD_CORES)'
+    substituteInPlace Makefile \
+      --replace 'BOOTSTRAP_ARGS :=' 'BOOTSTRAP_ARGS := --jobs $(NIX_BUILD_CORES)'
   '';
 
-  patches = patches ++ targetPatches;
+  patches = [
+    ./patches/net-tcp-disable-tests.patch
+
+    # Re-evaluate if this we need to disable this one
+    #./patches/stdsimd-disable-doctest.patch
+
+    # Fails on hydra - not locally; the exact reason is unknown.
+    # Comments in the test suggest that some non-reproducible environment
+    # variables such $RANDOM can make it fail.
+    # ./patches/disable-test-inherit-env.patch
+  ];
 
   # the rust build system complains that nix alters the checksums
   dontFixLibtool = true;
 
-  passthru.target = target;
-
   postPatch = ''
     patchShebangs src/etc
 
@@ -120,15 +156,13 @@ stdenv.mkDerivation {
   dontUseCmakeConfigure = true;
 
   # ps is needed for one of the test cases
-  nativeBuildInputs =
-    [ file python2 ps rustPlatform.rust.rustc git cmake
-      which libffi removeReferencesTo
-    ]
-    # Only needed for the debuginfo tests
+  nativeBuildInputs = [
+    file python2 ps rustPlatform.rust.rustc git cmake
+    which libffi removeReferencesTo
+  ] # Only needed for the debuginfo tests
     ++ optional (!stdenv.isDarwin) gdb;
 
-  buildInputs = targetToolchains
-    ++ optional stdenv.isDarwin Security
+  buildInputs = optional stdenv.isDarwin Security
     ++ optional (!withBundledLLVM) llvmShared;
 
   outputs = [ "out" "man" "doc" ];
@@ -148,7 +182,14 @@ stdenv.mkDerivation {
     sed -i '28s/home_dir().is_some()/true/' ./src/test/run-pass/env-home-dir.rs
   '';
 
-  inherit doCheck;
+  # 1. Upstream is not running tests on aarch64:
+  # see https://github.com/rust-lang/rust/issues/49807#issuecomment-380860567
+  # So we do the same.
+  # 2. Tests run out of memory for i686
+  #doCheck = !stdenv.isAarch64 && !stdenv.isi686;
+
+  # Disabled for now; see https://github.com/NixOS/nixpkgs/pull/42348#issuecomment-402115598.
+  doCheck = false;
 
   # remove references to llvm-config in lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/librustc_codegen_llvm-llvm.so
   # and thus a transitive dependency on ncurses
@@ -170,6 +211,5 @@ stdenv.mkDerivation {
     maintainers = with maintainers; [ madjar cstrahan wizeman globin havvy ];
     license = [ licenses.mit licenses.asl20 ];
     platforms = platforms.linux ++ platforms.darwin;
-    broken = broken;
   };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 690169095f9..632fd18bf46 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -7615,14 +7615,12 @@ in
     inherit (darwin) apple_sdk;
   };
 
-  # For beta and nightly releases use the nixpkgs-mozilla overlay
-  rust = callPackage ../development/compilers/rust ({
+  rust = callPackage ../development/compilers/rust {
     inherit (darwin.apple_sdk.frameworks) CoreFoundation Security;
-    llvm = llvm_7;
-  } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
-    stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
-  });
-  inherit (rust) cargo rustc;
+  };
+  rustPackages = rust.packages.stable;
+  inherit (rustPackages) cargo rustc rustPlatform;
+  inherit (rust) makeRustPlatform;
 
   buildRustCrate = callPackage ../build-support/rust/build-rust-crate { };
   buildRustCrateHelpers = callPackage ../build-support/rust/build-rust-crate/helpers.nix { };
@@ -7638,9 +7636,6 @@ in
 
   defaultCrateOverrides = callPackage ../build-support/rust/default-crate-overrides.nix { };
 
-  makeRustPlatform = callPackage ../build-support/rust/make-rust-platform.nix {};
-  rustPlatform = recurseIntoAttrs (makeRustPlatform rust);
-
   cargo-download = callPackage ../tools/package-management/cargo-download { };
   cargo-edit = callPackage ../tools/package-management/cargo-edit { };
   cargo-release = callPackage ../tools/package-management/cargo-release {