Python: improve cross-compilation

This changeset allows for cross-compilation of Python packages. Packages
built with buildPythonPackage are not allowed to refer to the build
machine. Executables that have shebangs will refer to the host.

14 files changed, 68 insertions, 41 deletions

diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix
index 2b383fe985d..0f8e088d434 100644
--- a/pkgs/development/interpreters/python/build-python-package-common.nix
+++ b/pkgs/development/interpreters/python/build-python-package-common.nix
@@ -1,7 +1,6 @@
 # This function provides generic bits to install a Python wheel.
 
 { python
-, bootstrapped-pip
 }:
 
 { buildInputs ? []
@@ -10,7 +9,7 @@
 , ... } @ attrs:
 
 attrs // {
-  buildInputs = buildInputs ++ [ bootstrapped-pip ];
+  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
 
   configurePhase = attrs.configurePhase or ''
     runHook preConfigure
@@ -24,7 +23,7 @@ attrs // {
     export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
 
     pushd dist
-    ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
+    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
     popd
 
     runHook postInstall
diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
index bc512357acd..4c66fdec5f6 100644
--- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix
+++ b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
@@ -2,7 +2,6 @@
 
 { lib
 , python
-, bootstrapped-pip
 }:
 
 {
@@ -26,13 +25,13 @@ in attrs // {
   buildPhase = attrs.buildPhase or ''
     runHook preBuild
     cp ${setuppy} nix_run_setup
-    ${python.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
+    ${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
     runHook postBuild
   '';
 
   installCheckPhase = attrs.checkPhase or ''
     runHook preCheck
-    ${python.interpreter} nix_run_setup test
+    ${python.pythonForBuild.interpreter} nix_run_setup test
     runHook postCheck
   '';
 
@@ -47,9 +46,9 @@ in attrs // {
     if test -e setup.py; then
       tmp_path=$(mktemp -d)
       export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.sitePackages}
-      ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
+      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
+      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
+      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
     fi
     ${postShellHook}
   '';
diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix
index 391086a662e..b664cf0b14f 100644
--- a/pkgs/development/interpreters/python/build-python-package.nix
+++ b/pkgs/development/interpreters/python/build-python-package.nix
@@ -10,17 +10,16 @@
 , ensureNewerSourcesForZipFilesHook
 , toPythonModule
 , namePrefix
-, bootstrapped-pip
 , flit
 , writeScript
 , update-python-libraries
 }:
 
 let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; };
+  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
   flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
   wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
+  common = import ./build-python-package-common.nix { inherit python; };
   mkPythonDerivation = import ./mk-python-derivation.nix {
     inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
     inherit toPythonModule namePrefix writeScript update-python-libraries;
diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 23b88b168b5..249c4ac9cf7 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -32,6 +32,9 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
+
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
     implementation = "cpython";
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index 0d1794f04fa..6e738a598dc 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -21,6 +21,7 @@
 , sourceVersion
 , sha256
 , passthruFun
+, bash
 }:
 
 assert x11Support -> tcl != null
@@ -46,7 +47,8 @@ let
   nativeBuildInputs = [
     nukeReferences
   ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-    buildPackages.stdenv.cc crossPython
+    buildPackages.stdenv.cc
+    pythonForBuild
   ];
 
   buildInputs = filter (p: p != null) [
@@ -56,11 +58,11 @@ let
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
 
-  crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
 
-  pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then
+  pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
     "$out/bin/python"
-  else crossPython.interpreter;
+  else pythonForBuild.interpreter;
 
 in with passthru; stdenv.mkDerivation {
   pname = "python3";
@@ -215,14 +217,25 @@ in with passthru; stdenv.mkDerivation {
     # We rebuild three times, once for each optimization level
     # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
     # This is automatically used when `SOURCE_DATE_EPOCH` is set.
-    find $out -name "*.py" | ${pythonForBuild}     -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -O  -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -OO -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter}     -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
+  '';
+
+  preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+    # Ensure patch-shebangs uses shebangs of host interpreter.
+    export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
   '';
 
   # Enforce that we don't have references to the OpenSSL -dev package, which we
   # explicitly specify in our configure flags above.
-  disallowedReferences = [ openssl.dev ];
+  disallowedReferences = [
+    openssl.dev
+  ] ++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+    # Ensure we don't have references to build-time packages.
+    # These typically end up in shebangs.
+    pythonForBuild buildPackages.bash
+  ];
 
   inherit passthru;
 
diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix
index 877bf647d26..786ae76c1dd 100644
--- a/pkgs/development/interpreters/python/default.nix
+++ b/pkgs/development/interpreters/python/default.nix
@@ -21,7 +21,6 @@ with pkgs;
         overrides = packageOverrides;
       };
     in rec {
-        sitePackages = "lib/${libPrefix}/site-packages";
         isPy27 = pythonVersion == "2.7";
         isPy33 = pythonVersion == "3.3"; # TODO: remove
         isPy34 = pythonVersion == "3.4"; # TODO: remove
@@ -35,7 +34,7 @@ with pkgs;
         withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;};
         pkgs = pythonPackages;
         interpreter = "${self}/bin/${executable}";
-        inherit executable implementation libPrefix pythonVersion;
+        inherit executable implementation libPrefix pythonVersion sitePackages;
         inherit sourceVersion;
         pythonAtLeast = lib.versionAtLeast pythonVersion;
         pythonOlder = lib.versionOlder pythonVersion;
@@ -112,8 +111,8 @@ in {
     inherit passthruFun;
   };
 
-  pypy3 = callPackage ./pypy {
-    self = pypy3;
+  pypy35 = callPackage ./pypy {
+    self = pypy35;
     sourceVersion = {
       major = "6";
       minor = "0";
diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
index b9a6835908f..7718bdfde5d 100644
--- a/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -77,7 +77,7 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
 
   buildInputs = [ wrapPython ]
     ++ lib.optional (lib.hasSuffix "zip" (attrs.src.name or "")) unzip
-    ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+#     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
     ++ buildInputs
     ++ pythonPath;
 
@@ -100,9 +100,12 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
     # Check if we have two packages with the same name in the closure and fail.
     # If this happens, something went wrong with the dependencies specs.
     # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
-    ${python.interpreter} ${./catch_conflicts}/catch_conflicts.py
+    ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
   '' + attrs.postFixup or '''';
 
+  # Python packages built through cross-compilation are always for the host platform.
+  disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
+
   meta = {
     # default to python's platforms
     platforms = python.meta.platforms;
diff --git a/pkgs/development/interpreters/python/wrap-python.nix b/pkgs/development/interpreters/python/wrap-python.nix
index 4ff0a62d7fb..6a19a215241 100644
--- a/pkgs/development/interpreters/python/wrap-python.nix
+++ b/pkgs/development/interpreters/python/wrap-python.nix
@@ -9,7 +9,8 @@ makeSetupHook {
       deps = makeWrapper;
       substitutions.sitePackages = python.sitePackages;
       substitutions.executable = python.interpreter;
-      substitutions.python = python;
+      substitutions.python = python.pythonForBuild;
+      substitutions.pythonHost = python;
       substitutions.magicalSedExpression = let
         # Looks weird? Of course, it's between single quoted shell strings.
         # NOTE: Order DOES matter here, so single character quotes need to be
diff --git a/pkgs/development/interpreters/python/wrap.sh b/pkgs/development/interpreters/python/wrap.sh
index 6fa8c316a17..b2d65422db4 100644
--- a/pkgs/development/interpreters/python/wrap.sh
+++ b/pkgs/development/interpreters/python/wrap.sh
@@ -16,8 +16,8 @@ buildPythonPath() {
     declare -A pythonPathsSeen=()
     program_PYTHONPATH=
     program_PATH=
-    pythonPathsSeen["@python@"]=1
-    addToSearchPath program_PATH @python@/bin
+    pythonPathsSeen["@pythonHost@"]=1
+    addToSearchPath program_PATH @pythonHost@/bin
     for path in $pythonPath; do
         _addToPythonPath $path
     done
@@ -53,7 +53,13 @@ wrapPythonProgramsIn() {
             # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
             # Python to use besides one with this hook anyway.
             if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then
-                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^"
+                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#!@executable@^"
+            fi
+
+            if head -n1 "$f" | grep -q '#!.*'; then
+                # Cross-compilation hack: ensure shebangs are for the host
+                echo "Rewriting $(head -n 1 $f) to #!@pythonHost@"
+                sed -i "$f" -e "1 s^#!@python@^#!@pythonHost@^"
             fi
 
             # catch /python and /.python-wrapped
diff --git a/pkgs/development/python-modules/cypari2/default.nix b/pkgs/development/python-modules/cypari2/default.nix
index a77e98dae4c..09326ad1a0b 100644
--- a/pkgs/development/python-modules/cypari2/default.nix
+++ b/pkgs/development/python-modules/cypari2/default.nix
@@ -1,5 +1,4 @@
 { stdenv
-, bootstrapped-pip
 , buildPythonPackage
 , python
 , fetchPypi
@@ -24,11 +23,11 @@ buildPythonPackage rec {
   # That is because while the default install phase succeeds to build the package,
   # it fails to generate the file "auto_paridecl.pxd".
   installPhase = ''
-    mkdir -p "$out/lib/${python.libPrefix}/site-packages"
-    export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
+    mkdir -p "$out/lib/${python.sitePackages}"
+    export PYTHONPATH="$out/lib/${python.sitePackages}:$PYTHONPATH"
 
     # install "." instead of "*.whl"
-    ${bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
+    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
   '';
 
   buildInputs = [
diff --git a/pkgs/development/python-modules/pyyaml/default.nix b/pkgs/development/python-modules/pyyaml/default.nix
index e66ca0df5b9..b4732e34c4e 100644
--- a/pkgs/development/python-modules/pyyaml/default.nix
+++ b/pkgs/development/python-modules/pyyaml/default.nix
@@ -1,4 +1,4 @@
-{ lib, buildPythonPackage, fetchPypi, libyaml }:
+{ lib, buildPythonPackage, fetchPypi, libyaml, buildPackages }:
 
 buildPythonPackage rec {
   pname = "PyYAML";
@@ -9,6 +9,8 @@ buildPythonPackage rec {
     sha256 = "3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf";
   };
 
+  nativeBuildInputs = [ buildPackages.stdenv.cc ];
+
   propagatedBuildInputs = [ libyaml ];
 
   meta = with lib; {
diff --git a/pkgs/development/python-modules/setuptools/default.nix b/pkgs/development/python-modules/setuptools/default.nix
index 09f848d456b..2663d6667e5 100644
--- a/pkgs/development/python-modules/setuptools/default.nix
+++ b/pkgs/development/python-modules/setuptools/default.nix
@@ -17,19 +17,24 @@ stdenv.mkDerivation rec {
     sha256 = "86bb4d8e1b0fabad1f4642b64c335b673e53e7a381de03c9a89fe678152c4c64";
   };
 
-  nativeBuildInputs = [ unzip wrapPython ];
-  buildInputs = [ python ];
+  nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ];
   doCheck = false;  # requires pytest
   installPhase = ''
       dst=$out/${python.sitePackages}
       mkdir -p $dst
       export PYTHONPATH="$dst:$PYTHONPATH"
-      ${python.interpreter} setup.py install --prefix=$out
+      ${python.pythonForBuild.interpreter} setup.py install --prefix=$out
       wrapPythonPrograms
   '';
 
   pythonPath = [];
 
+  dontPatchShebangs = true;
+
+  # Python packages built through cross-compilation are always for the host platform.
+  disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ];
+
+
   meta = with stdenv.lib; {
     description = "Utilities to facilitate the installation of Python packages";
     homepage = https://pypi.python.org/pypi/setuptools;
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 9b49b4f45e4..142173d4725 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -7937,6 +7937,7 @@ in
   python3 = python37;
   pypy = pypy2;
   pypy2 = pypy27;
+  pypy3 = pypy35;
 
   # Python interpreter that is build with all modules, including tkinter.
   # These are for compatibility and should not be used inside Nixpkgs.
@@ -7954,7 +7955,7 @@ in
   python3Packages = python3.pkgs;
 
   pythonInterpreters = callPackage ./../development/interpreters/python {};
-  inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy3;
+  inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy35;
 
   # Python package sets.
   python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs);
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index e0e45ea9ecd..d3aa62faadd 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -43,7 +43,6 @@ let
       else ff;
 
   buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
-    inherit bootstrapped-pip;
     flit = self.flit;
     # We want Python libraries to be named like e.g. "python3.6-${name}"
     inherit namePrefix;
@@ -51,7 +50,6 @@ let
   }));
 
   buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
-    inherit bootstrapped-pip;
     flit = self.flit;
     namePrefix = "";
     toPythonModule = x: x; # Application does not provide modules.