Python: improve cross-compilation

This changeset allows for cross-compilation of Python packages. Packages
built with buildPythonPackage are not allowed to refer to the build
machine. Executables that have shebangs will refer to the host.

9 files changed, 52 insertions, 30 deletions

diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix
index 2b383fe985d..0f8e088d434 100644
--- a/pkgs/development/interpreters/python/build-python-package-common.nix
+++ b/pkgs/development/interpreters/python/build-python-package-common.nix
@@ -1,7 +1,6 @@
 # This function provides generic bits to install a Python wheel.
 
 { python
-, bootstrapped-pip
 }:
 
 { buildInputs ? []
@@ -10,7 +9,7 @@
 , ... } @ attrs:
 
 attrs // {
-  buildInputs = buildInputs ++ [ bootstrapped-pip ];
+  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
 
   configurePhase = attrs.configurePhase or ''
     runHook preConfigure
@@ -24,7 +23,7 @@ attrs // {
     export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
 
     pushd dist
-    ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
+    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
     popd
 
     runHook postInstall
diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
index bc512357acd..4c66fdec5f6 100644
--- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix
+++ b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
@@ -2,7 +2,6 @@
 
 { lib
 , python
-, bootstrapped-pip
 }:
 
 {
@@ -26,13 +25,13 @@ in attrs // {
   buildPhase = attrs.buildPhase or ''
     runHook preBuild
     cp ${setuppy} nix_run_setup
-    ${python.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
+    ${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
     runHook postBuild
   '';
 
   installCheckPhase = attrs.checkPhase or ''
     runHook preCheck
-    ${python.interpreter} nix_run_setup test
+    ${python.pythonForBuild.interpreter} nix_run_setup test
     runHook postCheck
   '';
 
@@ -47,9 +46,9 @@ in attrs // {
     if test -e setup.py; then
       tmp_path=$(mktemp -d)
       export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.sitePackages}
-      ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
+      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
+      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
+      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
     fi
     ${postShellHook}
   '';
diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix
index 391086a662e..b664cf0b14f 100644
--- a/pkgs/development/interpreters/python/build-python-package.nix
+++ b/pkgs/development/interpreters/python/build-python-package.nix
@@ -10,17 +10,16 @@
 , ensureNewerSourcesForZipFilesHook
 , toPythonModule
 , namePrefix
-, bootstrapped-pip
 , flit
 , writeScript
 , update-python-libraries
 }:
 
 let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; };
+  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
   flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
   wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
+  common = import ./build-python-package-common.nix { inherit python; };
   mkPythonDerivation = import ./mk-python-derivation.nix {
     inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
     inherit toPythonModule namePrefix writeScript update-python-libraries;
diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 23b88b168b5..249c4ac9cf7 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -32,6 +32,9 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
+
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
     implementation = "cpython";
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index 0d1794f04fa..6e738a598dc 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -21,6 +21,7 @@
 , sourceVersion
 , sha256
 , passthruFun
+, bash
 }:
 
 assert x11Support -> tcl != null
@@ -46,7 +47,8 @@ let
   nativeBuildInputs = [
     nukeReferences
   ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-    buildPackages.stdenv.cc crossPython
+    buildPackages.stdenv.cc
+    pythonForBuild
   ];
 
   buildInputs = filter (p: p != null) [
@@ -56,11 +58,11 @@ let
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
 
-  crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
 
-  pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then
+  pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
     "$out/bin/python"
-  else crossPython.interpreter;
+  else pythonForBuild.interpreter;
 
 in with passthru; stdenv.mkDerivation {
   pname = "python3";
@@ -215,14 +217,25 @@ in with passthru; stdenv.mkDerivation {
     # We rebuild three times, once for each optimization level
     # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
     # This is automatically used when `SOURCE_DATE_EPOCH` is set.
-    find $out -name "*.py" | ${pythonForBuild}     -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -O  -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -OO -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter}     -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
+  '';
+
+  preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+    # Ensure patch-shebangs uses shebangs of host interpreter.
+    export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
   '';
 
   # Enforce that we don't have references to the OpenSSL -dev package, which we
   # explicitly specify in our configure flags above.
-  disallowedReferences = [ openssl.dev ];
+  disallowedReferences = [
+    openssl.dev
+  ] ++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+    # Ensure we don't have references to build-time packages.
+    # These typically end up in shebangs.
+    pythonForBuild buildPackages.bash
+  ];
 
   inherit passthru;
 
diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix
index 877bf647d26..786ae76c1dd 100644
--- a/pkgs/development/interpreters/python/default.nix
+++ b/pkgs/development/interpreters/python/default.nix
@@ -21,7 +21,6 @@ with pkgs;
         overrides = packageOverrides;
       };
     in rec {
-        sitePackages = "lib/${libPrefix}/site-packages";
         isPy27 = pythonVersion == "2.7";
         isPy33 = pythonVersion == "3.3"; # TODO: remove
         isPy34 = pythonVersion == "3.4"; # TODO: remove
@@ -35,7 +34,7 @@ with pkgs;
         withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;};
         pkgs = pythonPackages;
         interpreter = "${self}/bin/${executable}";
-        inherit executable implementation libPrefix pythonVersion;
+        inherit executable implementation libPrefix pythonVersion sitePackages;
         inherit sourceVersion;
         pythonAtLeast = lib.versionAtLeast pythonVersion;
         pythonOlder = lib.versionOlder pythonVersion;
@@ -112,8 +111,8 @@ in {
     inherit passthruFun;
   };
 
-  pypy3 = callPackage ./pypy {
-    self = pypy3;
+  pypy35 = callPackage ./pypy {
+    self = pypy35;
     sourceVersion = {
       major = "6";
       minor = "0";
diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
index b9a6835908f..7718bdfde5d 100644
--- a/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -77,7 +77,7 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
 
   buildInputs = [ wrapPython ]
     ++ lib.optional (lib.hasSuffix "zip" (attrs.src.name or "")) unzip
-    ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+#     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
     ++ buildInputs
     ++ pythonPath;
 
@@ -100,9 +100,12 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
     # Check if we have two packages with the same name in the closure and fail.
     # If this happens, something went wrong with the dependencies specs.
     # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
-    ${python.interpreter} ${./catch_conflicts}/catch_conflicts.py
+    ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
   '' + attrs.postFixup or '''';
 
+  # Python packages built through cross-compilation are always for the host platform.
+  disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
+
   meta = {
     # default to python's platforms
     platforms = python.meta.platforms;
diff --git a/pkgs/development/interpreters/python/wrap-python.nix b/pkgs/development/interpreters/python/wrap-python.nix
index 4ff0a62d7fb..6a19a215241 100644
--- a/pkgs/development/interpreters/python/wrap-python.nix
+++ b/pkgs/development/interpreters/python/wrap-python.nix
@@ -9,7 +9,8 @@ makeSetupHook {
       deps = makeWrapper;
       substitutions.sitePackages = python.sitePackages;
       substitutions.executable = python.interpreter;
-      substitutions.python = python;
+      substitutions.python = python.pythonForBuild;
+      substitutions.pythonHost = python;
       substitutions.magicalSedExpression = let
         # Looks weird? Of course, it's between single quoted shell strings.
         # NOTE: Order DOES matter here, so single character quotes need to be
diff --git a/pkgs/development/interpreters/python/wrap.sh b/pkgs/development/interpreters/python/wrap.sh
index 6fa8c316a17..b2d65422db4 100644
--- a/pkgs/development/interpreters/python/wrap.sh
+++ b/pkgs/development/interpreters/python/wrap.sh
@@ -16,8 +16,8 @@ buildPythonPath() {
     declare -A pythonPathsSeen=()
     program_PYTHONPATH=
     program_PATH=
-    pythonPathsSeen["@python@"]=1
-    addToSearchPath program_PATH @python@/bin
+    pythonPathsSeen["@pythonHost@"]=1
+    addToSearchPath program_PATH @pythonHost@/bin
     for path in $pythonPath; do
         _addToPythonPath $path
     done
@@ -53,7 +53,13 @@ wrapPythonProgramsIn() {
             # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
             # Python to use besides one with this hook anyway.
             if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then
-                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^"
+                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#!@executable@^"
+            fi
+
+            if head -n1 "$f" | grep -q '#!.*'; then
+                # Cross-compilation hack: ensure shebangs are for the host
+                echo "Rewriting $(head -n 1 $f) to #!@pythonHost@"
+                sed -i "$f" -e "1 s^#!@python@^#!@pythonHost@^"
             fi
 
             # catch /python and /.python-wrapped