diff options
28 files changed, 500 insertions, 294 deletions
diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 4963c97a6c9..88dc42ebc6c 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -540,7 +540,8 @@ and the aliases #### `buildPythonPackage` function The `buildPythonPackage` function is implemented in -`pkgs/development/interpreters/python/build-python-package.nix` +`pkgs/development/interpreters/python/mk-python-derivation` +using setup hooks. The following is an example: ```nix @@ -797,6 +798,22 @@ such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv` should be used with `ignoreCollisions = true`. +#### Setup hooks + +The following are setup hooks specifically for Python packages. Most of these are +used in `buildPythonPackage`. + +- `flitBuildHook` to build a wheel using `flit`. +- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`. +- `pipInstallHook` to install wheels. +- `pytestCheckHook` to run tests with `pytest`. +- `pythonCatchConflictsHook` to check whether a Python package is not already existing. +- `pythonImportsCheckHook` to check whether importing the listed modules works. +- `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder. +- `setuptoolsBuildHook` to build a wheel using `setuptools`. +- `setuptoolsCheckHook` to run tests with `python setup.py test`. +- `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`. + ### Development mode Development or editable mode is supported. To develop Python packages diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix deleted file mode 100644 index 0f8e088d434..00000000000 --- a/pkgs/development/interpreters/python/build-python-package-common.nix +++ /dev/null @@ -1,31 +0,0 @@ -# This function provides generic bits to install a Python wheel. - -{ python -}: - -{ buildInputs ? [] -# Additional flags to pass to "pip install". -, installFlags ? [] -, ... } @ attrs: - -attrs // { - buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ]; - - configurePhase = attrs.configurePhase or '' - runHook preConfigure - runHook postConfigure - ''; - - installPhase = attrs.installPhase or '' - runHook preInstall - - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" - - pushd dist - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild - popd - - runHook postInstall - ''; -} diff --git a/pkgs/development/interpreters/python/build-python-package-flit.nix b/pkgs/development/interpreters/python/build-python-package-flit.nix deleted file mode 100644 index b0f9e038021..00000000000 --- a/pkgs/development/interpreters/python/build-python-package-flit.nix +++ /dev/null @@ -1,22 +0,0 @@ -# This function provides specific bits for building a flit-based Python package. - -{ python -, flit -}: - -{ ... } @ attrs: - -attrs // { - nativeBuildInputs = [ flit ]; - buildPhase = attrs.buildPhase or '' - runHook preBuild - flit build --format wheel - runHook postBuild - ''; - - # Flit packages, like setuptools packages, might have tests. - installCheckPhase = attrs.checkPhase or '' - ${python.interpreter} -m unittest discover - ''; - doCheck = attrs.doCheck or true; -} diff --git a/pkgs/development/interpreters/python/build-python-package-pyproject.nix b/pkgs/development/interpreters/python/build-python-package-pyproject.nix deleted file mode 100644 index 085db44f3e8..00000000000 --- a/pkgs/development/interpreters/python/build-python-package-pyproject.nix +++ /dev/null @@ -1,56 +0,0 @@ -# This function provides specific bits for building a setuptools-based Python package. - -{ lib -, python -}: - -{ -# Global options passed to "python setup.py" - setupPyGlobalFlags ? [] -# Build options passed to "build_ext" -# https://github.com/pypa/pip/issues/881 -# Rename to `buildOptions` because it is not setuptools specific? -, setupPyBuildFlags ? [] -# Execute before shell hook -, preShellHook ? "" -# Execute after shell hook -, postShellHook ? "" -, ... } @ attrs: - -let - pipGlobalFlagsString = lib.concatMapStringsSep " " (option: "--global-option ${option}") setupPyGlobalFlags; - pipBuildFlagsString = lib.concatMapStringsSep " " (option: "--build-option ${option}") setupPyBuildFlags; -in attrs // { - buildPhase = attrs.buildPhase or '' - runHook preBuild - mkdir -p dist - echo "Creating a wheel..." - ${python.pythonForBuild.interpreter} -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist ${pipGlobalFlagsString} ${pipBuildFlagsString} . - echo "Finished creating a wheel..." - runHook postBuild - ''; - - installCheckPhase = '' - runHook preCheck - echo "No checkPhase defined. Either provide a checkPhase or disable tests in case tests are not available."; exit 1 - runHook postCheck - ''; - - # With Python it's a common idiom to run the tests - # after the software has been installed. - doCheck = attrs.doCheck or true; - - shellHook = attrs.shellHook or '' - ${preShellHook} - # Long-term setup.py should be dropped. - if [ -e pyproject.toml ]; then - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${python.pythonForBuild.sitePackages} - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2 - fi - ${postShellHook} - ''; - -} diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix deleted file mode 100644 index 7738ea2f66a..00000000000 --- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix +++ /dev/null @@ -1,60 +0,0 @@ -# This function provides specific bits for building a setuptools-based Python package. - -{ lib -, python -}: - -{ -# Global options passed to "python setup.py" - setupPyGlobalFlags ? [] -# Build options passed to "python setup.py build_ext" -# https://github.com/pypa/pip/issues/881 -, setupPyBuildFlags ? [] -# Execute before shell hook -, preShellHook ? "" -# Execute after shell hook -, postShellHook ? "" -, ... } @ attrs: - -let - # use setuptools shim (so that setuptools is imported before distutils) - # pip does the same thing: https://github.com/pypa/pip/pull/3265 - setuppy = ./run_setup.py; - - setupPyGlobalFlagsString = lib.concatStringsSep " " setupPyGlobalFlags; - setupPyBuildExtString = lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags)); - -in attrs // { - # we copy nix_run_setup over so it's executed relative to the root of the source - # many project make that assumption - buildPhase = attrs.buildPhase or '' - runHook preBuild - cp ${setuppy} nix_run_setup - ${python.pythonForBuild.interpreter} nix_run_setup ${setupPyGlobalFlagsString} ${setupPyBuildExtString} bdist_wheel - runHook postBuild - ''; - - installCheckPhase = attrs.checkPhase or '' - runHook preCheck - ${python.pythonForBuild.interpreter} nix_run_setup test - runHook postCheck - ''; - - # Python packages that are installed with setuptools - # are typically distributed with tests. - # With Python it's a common idiom to run the tests - # after the software has been installed. - doCheck = attrs.doCheck or true; - - shellHook = attrs.shellHook or '' - ${preShellHook} - if test -e setup.py; then - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${python.pythonForBuild.sitePackages} - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2 - fi - ${postShellHook} - ''; -} diff --git a/pkgs/development/interpreters/python/build-python-package-wheel.nix b/pkgs/development/interpreters/python/build-python-package-wheel.nix deleted file mode 100644 index e3c4e13c0e2..00000000000 --- a/pkgs/development/interpreters/python/build-python-package-wheel.nix +++ /dev/null @@ -1,20 +0,0 @@ -# This function provides specific bits for building a wheel-based Python package. - -{ -}: - -{ ... } @ attrs: - -attrs // { - unpackPhase = '' - mkdir dist - cp "$src" "dist/$(stripHash "$src")" - ''; - - # Wheels are pre-compiled - buildPhase = attrs.buildPhase or ":"; - installCheckPhase = attrs.checkPhase or ":"; - - # Wheels don't have any checks to run - doCheck = attrs.doCheck or false; -} \ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix deleted file mode 100644 index 61c1186cef9..00000000000 --- a/pkgs/development/interpreters/python/build-python-package.nix +++ /dev/null @@ -1,48 +0,0 @@ -# This function provides a generic Python package builder, -# and can build packages that use distutils, setuptools or flit. - -{ lib -, config -, python -, wrapPython -, setuptools -, unzip -, ensureNewerSourcesForZipFilesHook -, toPythonModule -, namePrefix -, flit -, writeScript -, update-python-libraries -}: - -let - setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; }; - pyproject-specific = import ./build-python-package-pyproject.nix { inherit lib python; }; - flit-specific = import ./build-python-package-flit.nix { inherit python flit; }; - wheel-specific = import ./build-python-package-wheel.nix { }; - common = import ./build-python-package-common.nix { inherit python; }; - mkPythonDerivation = import ./mk-python-derivation.nix { - inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook; - inherit toPythonModule namePrefix update-python-libraries; - }; -in - -{ -# Several package formats are supported. -# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. -# "wheel" : Install from a pre-compiled wheel. -# "flit" : Install a flit package. This builds a wheel. -# "other" : Provide your own buildPhase and installPhase. -format ? "setuptools" -, ... } @ attrs: - -let - formatspecific = - if format == "pyproject" then common (pyproject-specific attrs) - else if format == "setuptools" then common (setuptools-specific attrs) - else if format == "flit" then common (flit-specific attrs) - else if format == "wheel" then common (wheel-specific attrs) - else if format == "other" then {} - else throw "Unsupported format ${format}"; - -in mkPythonDerivation ( attrs // formatspecific ) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix new file mode 100644 index 00000000000..9a7ec98ba17 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -0,0 +1,95 @@ +# Hooks for building Python packages. +{ python +, callPackage +, makeSetupHook +}: + +let + pythonInterpreter = python.pythonForBuild.interpreter; + pythonSitePackages = python.sitePackages; + pythonCheckInterpreter = python.interpreter; + setuppy = ../run_setup.py; +in rec { + + flitBuildHook = callPackage ({ flit }: + makeSetupHook { + name = "flit-build-hook"; + deps = [ flit ]; + substitutions = { + inherit pythonInterpreter; + }; + } ./flit-build-hook.sh) {}; + + pipBuildHook = callPackage ({ pip }: + makeSetupHook { + name = "pip-build-hook.sh"; + deps = [ pip ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pip-build-hook.sh) {}; + + pipInstallHook = callPackage ({ pip }: + makeSetupHook { + name = "pip-install-hook"; + deps = [ pip ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pip-install-hook.sh) {}; + + pytestCheckHook = callPackage ({ pytest }: + makeSetupHook { + name = "pytest-check-hook"; + deps = [ pytest ]; + substitutions = { + inherit pythonCheckInterpreter; + }; + } ./pytest-check-hook.sh) {}; + + pythonCatchConflictsHook = callPackage ({ setuptools }: + makeSetupHook { + name = "python-catch-conflicts-hook"; + deps = [ setuptools ]; + substitutions = { + inherit pythonInterpreter; + catchConflicts=../catch_conflicts/catch_conflicts.py; + }; + } ./python-catch-conflicts-hook.sh) {}; + + pythonImportsCheckHook = callPackage ({}: + makeSetupHook { + name = "python-imports-check-hook.sh"; + substitutions = { + inherit pythonCheckInterpreter; + }; + } ./python-imports-check-hook.sh) {}; + + pythonRemoveBinBytecodeHook = callPackage ({ }: + makeSetupHook { + name = "python-remove-bin-bytecode-hook"; + } ./python-remove-bin-bytecode-hook.sh) {}; + + setuptoolsBuildHook = callPackage ({ setuptools, wheel }: + makeSetupHook { + name = "setuptools-setup-hook"; + deps = [ setuptools wheel ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages setuppy; + }; + } ./setuptools-build-hook.sh) {}; + + setuptoolsCheckHook = callPackage ({ setuptools }: + makeSetupHook { + name = "setuptools-check-hook"; + deps = [ setuptools ]; + substitutions = { + inherit pythonCheckInterpreter setuppy; + }; + } ./setuptools-check-hook.sh) {}; + + wheelUnpackHook = callPackage ({ }: + makeSetupHook { + name = "wheel-unpack-hook.sh"; + } ./wheel-unpack-hook.sh) {}; +} diff --git a/pkgs/development/interpreters/python/hooks/flit-build-hook.sh b/pkgs/development/interpreters/python/hooks/flit-build-hook.sh new file mode 100644 index 00000000000..faa3f6e3075 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/flit-build-hook.sh @@ -0,0 +1,15 @@ +# Setup hook for flit +echo "Sourcing flit-build-hook" + +flitBuildPhase () { + echo "Executing flitBuildPhase" + preBuild + @pythonInterpreter@ -m flit build --format wheel + postBuild + echo "Finished executing flitBuildPhase" +} + +if [ -z "$dontUseFlitBuild" ] && [ -z "$buildPhase" ]; then + echo "Using flitBuildPhase" + buildPhase=flitBuildPhase +fi diff --git a/pkgs/development/interpreters/python/hooks/pip-build-hook.sh b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh new file mode 100644 index 00000000000..6796d3efd0a --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh @@ -0,0 +1,42 @@ +# Setup hook to use for pip projects +echo "Sourcing pip-build-hook" + +pipBuildPhase() { + echo "Executing pipBuildPhase" + runHook preBuild + + mkdir -p dist + echo "Creating a wheel..." + @pythonInterpreter@ -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist "$options" . + echo "Finished creating a wheel..." + + runHook postBuild + echo "Finished executing pipBuildPhase" +} + +pipShellHook() { + echo "Executing pipShellHook" + runHook preShellHook + + # Long-term setup.py should be dropped. + if [ -e pyproject.toml ]; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH" + mkdir -p "$tmp_path/@pythonSitePackages@" + @pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2 + fi + + runHook postShellHook + echo "Finished executing pipShellHook" +} + +if [ -z "$dontUsePipBuild" ] && [ -z "$buildPhase" ]; then + echo "Using pipBuildPhase" + buildPhase=pipBuildPhase +fi + +if [ -z "$shellHook" ]; then + echo "Using pipShellHook" + shellHook=pipShellHook +fi diff --git a/pkgs/development/interpreters/python/hooks/pip-install-hook.sh b/pkgs/development/interpreters/python/hooks/pip-install-hook.sh new file mode 100644 index 00000000000..f528ec63cb8 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/pip-install-hook.sh @@ -0,0 +1,24 @@ +# Setup hook for pip. +echo "Sourcing pip-install-hook" + +declare -a pipInstallFlags + +pipInstallPhase() { + echo "Executing pipInstallPhase" + runHook preInstall + + mkdir -p "$out/@pythonSitePackages@" + export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH" + + pushd dist || return 1 + @pythonInterpreter@ -m pip install ./*.whl --no-index --prefix="$out" --no-cache $pipInstallFlags --build tmpbuild + popd || return 1 + + runHook postInstall + echo "Finished executing pipInstallPhase" +} + +if [ -z "$dontUsePipInstall" ] && [ -z "$installPhase" ]; then + echo "Using pipInstallPhase" + installPhase=pipInstallPhase +fi diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh new file mode 100644 index 00000000000..24510b9f993 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh @@ -0,0 +1,49 @@ +# Setup hook for pytest +echo "Sourcing pytest-check-hook" + +declare -ar disabledTests + +function _concatSep { + local result + local sep="$1" + local -n arr=$2 + for index in ${!arr[*]}; do + if [ $index -eq 0 ]; then + result="${arr[index]}" + else + result+=" $sep ${arr[index]}" + fi + done + echo "$result" +} + +function _pytestComputeDisabledTestsString () { + declare -a tests + local tests=($1) + local prefix="not " + prefixed=( "${tests[@]/#/$prefix}" ) + result=$(_concatSep "and" prefixed) + echo "$result" +} + +function pytestCheckPhase() { + echo "Executing pytestCheckPhase" + runHook preCheck + + # Compose arguments + args=" -m pytest" + if [ -n "$disabledTests" ]; then + disabledTestsString=$(_pytestComputeDisabledTestsString "${disabledTests[@]}") + args+=" -k \""$disabledTestsString"\"" + fi + args+=" ${pytestFlagsArray[@]}" + eval "@pythonCheckInterpreter@ $args" + + runHook postCheck + echo "Finished executing pytestCheckPhase" +} + +if [ -z "$dontUsePytestCheck" ] && [ -z "$installCheckPhase" ]; then + echo "Using pytestCheckPhase" + preDistPhases+=" pytestCheckPhase" +fi diff --git a/pkgs/development/interpreters/python/hooks/python-catch-conflicts-hook.sh b/pkgs/development/interpreters/python/hooks/python-catch-conflicts-hook.sh new file mode 100644 index 00000000000..e9065cf1793 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/python-catch-conflicts-hook.sh @@ -0,0 +1,10 @@ +# Setup hook for detecting conflicts in Python packages +echo "Sourcing python-catch-conflicts-hook.sh" + +pythonCatchConflictsPhase() { + @pythonInterpreter@ @catchConflicts@ +} + +if [ -z "$dontUsePythonCatchConflicts" ]; then + preDistPhases+=" pythonCatchConflictsPhase" +fi diff --git a/pkgs/development/interpreters/python/hooks/python-imports-check-hook.sh b/pkgs/development/interpreters/python/hooks/python-imports-check-hook.sh new file mode 100644 index 00000000000..7e2b3f69d6d --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/python-imports-check-hook.sh @@ -0,0 +1,16 @@ +# Setup hook for checking whether Python imports succeed +echo "Sourcing python-imports-check-hook.sh" + +pythonImportsCheckPhase () { + echo "Executing pythonImportsCheckPhase" + + if [ -n "$pythonImportsCheck" ]; then + echo "Check whether the following modules can be imported: $pythonImportsCheck" + cd $out && eval "@pythonCheckInterpreter@ -c 'import os; import importlib; list(map(lambda mod: importlib.import_module(mod), os.environ[\"pythonImportsCheck\"].split()))'" + fi +} + +if [ -z "$dontUsePythonImportsCheck" ]; then + echo "Using pythonImportsCheckPhase" + preDistPhases+=" pythonImportsCheckPhase" +fi diff --git a/pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh b/pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh new file mode 100644 index 00000000000..960de767be7 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh @@ -0,0 +1,17 @@ +# Setup hook for detecting conflicts in Python packages +echo "Sourcing python-remove-bin-bytecode-hook.sh" + +# Check if we have two packages with the same name in the closure and fail. +# If this happens, something went wrong with the dependencies specs. +# Intentionally kept in a subdirectory, see catch_conflicts/README.md. + +pythonRemoveBinBytecodePhase () { + if [ -d "$out/bin" ]; then + rm -rf "$out/bin/__pycache__" # Python 3 + find "$out/bin" -type f -name "*.pyc" -delete # Python 2 + fi +} + +if [ -z "$dontUsePythonRemoveBinBytecode" ]; then + preDistPhases+=" pythonRemoveBinBytecodePhase" +fi diff --git a/pkgs/development/interpreters/python/hooks/setuptools-build-hook.sh b/pkgs/development/interpreters/python/hooks/setuptools-build-hook.sh new file mode 100644 index 00000000000..db3e4225d29 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/setuptools-build-hook.sh @@ -0,0 +1,47 @@ +# Setup hook for setuptools. +echo "Sourcing setuptools-build-hook" + +setuptoolsBuildPhase() { + echo "Executing setuptoolsBuildPhase" + local args + runHook preBuild + + cp -f @setuppy@ nix_run_setup + args="" + if [ -n "$setupPyGlobalFlags" ]; then + args+="$setupPyGlobalFlags" + fi + if [ -n "$setupPyBuildFlags" ]; then + args+="build_ext $setupPyBuildFlags" + fi + eval "@pythonInterpreter@ nix_run_setup $args bdist_wheel" + + runHook postBuild + echo "Finished executing setuptoolsInstallPhase" +} + +setuptoolsShellHook() { + echo "Executing setuptoolsShellHook" + runHook preShellHook + + if test -e setup.py; then + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="@pythonSitePackages@:$PYTHONPATH" + mkdir -p "$tmp_path/@pythonSitePackages@" + eval "@pythonInterpreter@ -m pip -e . --prefix $tmp_path >&2" + fi + + runHook postShellHook + echo "Finished executing setuptoolsShellHook" +} + +if [ -z "$dontUseSetuptoolsBuild" ] && [ -z "$buildPhase" ]; then + echo "Using setuptoolsBuildPhase" + buildPhase=setuptoolsBuildPhase +fi + +if [ -z "$dontUseSetuptoolsShellHook" ] && [ -z "$shellHook" ]; then + echo "Using setuptoolsShellHook" + shellHook=setuptoolsShellHook +fi diff --git a/pkgs/development/interpreters/python/hooks/setuptools-check-hook.sh b/pkgs/development/interpreters/python/hooks/setuptools-check-hook.sh new file mode 100644 index 00000000000..71bb036a91a --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/setuptools-check-hook.sh @@ -0,0 +1,18 @@ +# Setup hook for setuptools. +echo "Sourcing setuptools-check-hook" + +setuptoolsCheckPhase() { + echo "Executing setuptoolsCheckPhase" + runHook preCheck + + cp -f @setuppy@ nix_run_setup + @pythonCheckInterpreter@ nix_run_setup test + + runHook postCheck + echo "Finished executing setuptoolsCheckPhase" +} + +if [ -z "$dontUseSetuptoolsCheck" ] && [ -z "$installCheckPhase" ]; then + echo "Using setuptoolsCheckPhase" + preDistPhases+=" setuptoolsCheckPhase" +fi diff --git a/pkgs/development/interpreters/python/hooks/wheel-unpack-hook.sh b/pkgs/development/interpreters/python/hooks/wheel-unpack-hook.sh new file mode 100644 index 00000000000..6dd0c5be4cb --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/wheel-unpack-hook.sh @@ -0,0 +1,18 @@ +# Setup hook to use in case a wheel is fetched +echo "Sourcing wheel setup hook" + +wheelUnpackPhase(){ + echo "Executing wheelUnpackPhase" + runHook preUnpack + + mkdir -p dist + cp "$src" "dist/$(stripHash "$src")" + +# runHook postUnpack # Calls find...? + echo "Finished executing wheelUnpackPhase" +} + +if [ -z "$dontUseWheelUnpack" ] && [ -z "$unpackPhase" ]; then + echo "Using wheelUnpackPhase" + unpackPhase=wheelUnpackPhase +fi diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 6a9e3d48bdb..700894eda6d 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -4,13 +4,22 @@ , config , python , wrapPython -, setuptools , unzip , ensureNewerSourcesForZipFilesHook # Whether the derivation provides a Python module or not. , toPythonModule , namePrefix , update-python-libraries +, setuptools +, flitBuildHook +, pipBuildHook +, pipInstallHook +, pythonCatchConflictsHook +, pythonImportsCheckHook +, pythonRemoveBinBytecodeHook +, setuptoolsBuildHook +, setuptoolsCheckHook +, wheelUnpackHook }: { name ? "${attrs.pname}-${attrs.version}" @@ -48,6 +57,11 @@ # Skip wrapping of python programs altogether , dontWrapPythonPrograms ? false +# Don't use Pip to install a wheel +# Note this is actually a variable for the pipInstallPhase in pip's setupHook. +# It's included here to prevent an infinite recursion. +, dontUsePipInstall ? false + # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs , permitUserSite ? false @@ -57,6 +71,13 @@ # However, some packages do provide executables with extensions, and thus bytecode is generated. , removeBinBytecode ? true +# Several package formats are supported. +# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. +# "wheel" : Install from a pre-compiled wheel. +# "flit" : Install a flit package. This builds a wheel. +# "other" : Provide your own buildPhase and installPhase. +, format ? "setuptools" + , meta ? {} , passthru ? {} @@ -71,26 +92,43 @@ if disabled then throw "${name} not supported for interpreter ${python.executable}" else -let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attrs [ - "disabled" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" - ] // { +let + inherit (python) stdenv; + + self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ + "disabled" "checkPhase" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format" + ]) // { name = namePrefix + name; nativeBuildInputs = [ python wrapPython - ensureNewerSourcesForZipFilesHook - setuptools -# ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools + ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)? + ] ++ lib.optionals catchConflicts [ + setuptools pythonCatchConflictsHook + ] ++ lib.optionals removeBinBytecode [ + pythonRemoveBinBytecodeHook ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [ unzip + ] ++ lib.optionals (format == "setuptools") [ + setuptoolsBuildHook + ] ++ lib.optionals (format == "flit") [ + flitBuildHook + ] ++ lib.optionals (format == "pyproject") [ + pipBuildHook + ] ++ lib.optionals (format == "wheel") [ + wheelUnpackHook + ] ++ lib.optionals (!(format == "other") || dontUsePipInstall) [ + pipInstallHook + ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ + # This is a test, however, it should be ran independent of the checkPhase and checkInputs + pythonImportsCheckHook ] ++ nativeBuildInputs; buildInputs = buildInputs ++ pythonPath; - # Propagate python and setuptools. We should stop propagating setuptools. - propagatedBuildInputs = propagatedBuildInputs ++ [ python setuptools ]; + propagatedBuildInputs = propagatedBuildInputs ++ [ python ]; inherit strictDeps; @@ -98,21 +136,17 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr # Python packages don't have a checkPhase, only an installCheckPhase doCheck = false; - doInstallCheck = doCheck; - installCheckInputs = checkInputs; + doInstallCheck = attrs.doCheck or true; + installCheckInputs = [ + ] ++ lib.optionals (format == "setuptools") [ + # Longer-term we should get rid of this and require + # users of this function to set the `installCheckPhase` or + # pass in a hook that sets it. + setuptoolsCheckHook + ] ++ checkInputs; postFixup = lib.optionalString (!dontWrapPythonPrograms) '' wrapPythonPrograms - '' + lib.optionalString removeBinBytecode '' - if [ -d "$out/bin" ]; then - rm -rf "$out/bin/__pycache__" # Python 3 - find "$out/bin" -type f -name "*.pyc" -delete # Python 2 - fi - '' + lib.optionalString catchConflicts '' - # Check if we have two packages with the same name in the closure and fail. - # If this happens, something went wrong with the dependencies specs. - # Intentionally kept in a subdirectory, see catch_conflicts/README.md. - ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py '' + attrs.postFixup or ''''; # Python packages built through cross-compilation are always for the host platform. @@ -123,6 +157,10 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr platforms = python.meta.platforms; isBuildPythonPackage = python.meta.platforms; } // meta; +} // lib.optionalAttrs (attrs?checkPhase) { + # If given use the specified checkPhase, otherwise use the setup hook. + # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`. + installCheckPhase = attrs.checkPhase; })); passthru.updateScript = let diff --git a/pkgs/development/python-modules/atomicwrites/default.nix b/pkgs/development/python-modules/atomicwrites/default.nix index e34f937b509..eed9591d7bd 100644 --- a/pkgs/development/python-modules/atomicwrites/default.nix +++ b/pkgs/development/python-modules/atomicwrites/default.nix @@ -1,4 +1,4 @@ -{ stdenv, buildPythonPackage, fetchPypi }: +{ stdenv, buildPythonPackage, fetchPypi, pytest }: buildPythonPackage rec { pname = "atomicwrites"; @@ -9,6 +9,10 @@ buildPythonPackage rec { sha256 = "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"; }; + # Tests depend on pytest but atomicwrites is a dependency of pytest + doCheck = false; + checkInputs = [ pytest ]; + meta = with stdenv.lib; { description = "Atomic file writes on POSIX"; homepage = https://pypi.python.org/pypi/atomicwrites; diff --git a/pkgs/development/python-modules/bootstrapped-pip/default.nix b/pkgs/development/python-modules/bootstrapped-pip/default.nix index 1455a783593..3039c8fa1b9 100644 --- a/pkgs/development/python-modules/bootstrapped-pip/default.nix +++ b/pkgs/development/python-modules/bootstrapped-pip/default.nix @@ -1,4 +1,8 @@ -{ stdenv, python, fetchPypi, makeWrapper, unzip }: +{ stdenv, python, fetchPypi, makeWrapper, unzip, makeSetupHook +, pipInstallHook +, setuptoolsBuildHook + +}: let wheel_source = fetchPypi { @@ -25,6 +29,15 @@ in stdenv.mkDerivation rec { sha256 = "993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676"; }; + dontUseSetuptoolsBuild = true; + + # Should be propagatedNativeBuildInputs + propagatedBuildInputs = [ + # Override to remove dependencies to prevent infinite recursion. + (pipInstallHook.override{pip=null;}) + (setuptoolsBuildHook.override{setuptools=null; wheel=null;}) + ]; + unpackPhase = '' mkdir -p $out/${python.sitePackages} unzip -d $out/${python.sitePackages} $src @@ -32,7 +45,7 @@ in stdenv.mkDerivation rec { unzip -d $out/${python.sitePackages} ${wheel_source} ''; - patchPhase = '' + postPatch = '' mkdir -p $out/bin ''; @@ -52,4 +65,5 @@ in stdenv.mkDerivation rec { wrapProgram $f --prefix PYTHONPATH ":" $out/${python.sitePackages}/ done ''; + } diff --git a/pkgs/development/python-modules/pip/default.nix b/pkgs/development/python-modules/pip/default.nix index 76f3b0b7176..00159449d05 100644 --- a/pkgs/development/python-modules/pip/default.nix +++ b/pkgs/development/python-modules/pip/default.nix @@ -1,25 +1,32 @@ { lib +, python , buildPythonPackage +, bootstrapped-pip , fetchPypi , mock , scripttest , virtualenv , pretend , pytest +, setuptools +, wheel }: buildPythonPackage rec { pname = "pip"; version = "19.1.1"; + format = "other"; src = fetchPypi { inherit pname version; sha256 = "44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958"; }; + nativeBuildInputs = [ bootstrapped-pip ]; + # pip detects that we already have bootstrapped_pip "installed", so we need # to force it a little. - installFlags = [ "--ignore-installed" ]; + pipInstallFlags = [ "--ignore-installed" ]; checkInputs = [ mock scripttest virtualenv pretend pytest ]; # Pip wants pytest, but tests are not distributed diff --git a/pkgs/development/python-modules/py/default.nix b/pkgs/development/python-modules/py/default.nix index e54fd1521b3..9c5ada22b14 100644 --- a/pkgs/development/python-modules/py/default.nix +++ b/pkgs/development/python-modules/py/default.nix @@ -12,7 +12,11 @@ buildPythonPackage rec { # Circular dependency on pytest doCheck = false; - buildInputs = [ setuptools_scm ]; + nativeBuildInputs = [ setuptools_scm ]; + + pythonImportsCheck = [ + "py" + ]; meta = with stdenv.lib; { description = "Library with cross-python path, ini-parsing, io, code, log facilities"; diff --git a/pkgs/development/python-modules/pytest/default.nix b/pkgs/development/python-modules/pytest/default.nix index 7866454a62f..506b560e01a 100644 --- a/pkgs/development/python-modules/pytest/default.nix +++ b/pkgs/development/python-modules/pytest/default.nix @@ -1,6 +1,6 @@ { stdenv, buildPythonPackage, pythonOlder, fetchPypi, attrs, hypothesis, py , setuptools_scm, setuptools, six, pluggy, funcsigs, isPy3k, more-itertools -, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy +, atomicwrites, mock, writeText, pathlib2, wcwidth, packaging, isPyPy, python }: buildPythonPackage rec { version = "5.1.0"; @@ -17,12 +17,13 @@ buildPythonPackage rec { }; checkInputs = [ hypothesis mock ]; - buildInputs = [ setuptools_scm ]; + nativeBuildInputs = [ setuptools_scm ]; propagatedBuildInputs = [ attrs py setuptools six pluggy more-itertools atomicwrites wcwidth packaging ] ++ stdenv.lib.optionals (!isPy3k) [ funcsigs ] ++ stdenv.lib.optionals (pythonOlder "3.6") [ pathlib2 ]; doCheck = !isPyPy; # https://github.com/pytest-dev/pytest/issues/3460 + # Ignored file https://github.com/pytest-dev/pytest/pull/5605#issuecomment-522243929 checkPhase = '' runHook preCheck @@ -35,15 +36,17 @@ buildPythonPackage rec { pytestcachePhase() { find $out -name .pytest_cache -type d -exec rm -rf {} + } - preDistPhases+=" pytestcachePhase" ''; + pythonImportsCheck = [ + "pytest" + ]; + meta = with stdenv.lib; { homepage = https://docs.pytest.org; description = "Framework for writing tests"; maintainers = with maintainers; [ domenkozar lovek323 madjar lsix ]; license = licenses.mit; - platforms = platforms.unix; }; } diff --git a/pkgs/development/python-modules/setuptools/default.nix b/pkgs/development/python-modules/setuptools/default.nix index a849dad54aa..8506b0f28d5 100644 --- a/pkgs/development/python-modules/setuptools/default.nix +++ b/pkgs/development/python-modules/setuptools/default.nix @@ -1,15 +1,17 @@ { stdenv +, buildPythonPackage , fetchPypi , python , wrapPython , unzip +, callPackage +, bootstrapped-pip }: -# Should use buildPythonPackage here somehow -stdenv.mkDerivation rec { +buildPythonPackage rec { pname = "setuptools"; version = "41.0.1"; - name = "${python.libPrefix}-${pname}-${version}"; + format = "other"; src = fetchPypi { inherit pname version; @@ -17,8 +19,11 @@ stdenv.mkDerivation rec { sha256 = "a222d126f5471598053c9a77f4b5d4f26eaa1f150ad6e01dcf1a42e185d05613"; }; - nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ]; - doCheck = false; # requires pytest + # There is nothing to build + dontBuild = true; + + nativeBuildInputs = [ bootstrapped-pip ]; + installPhase = '' dst=$out/${python.sitePackages} mkdir -p $dst @@ -27,13 +32,11 @@ stdenv.mkDerivation rec { wrapPythonPrograms ''; - pythonPath = []; - - dontPatchShebangs = true; - - # Python packages built through cross-compilation are always for the host platform. - disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ]; + # Adds setuptools to nativeBuildInputs causing infinite recursion. + catchConflicts = false; + # Requires pytest, causing infinite recursion. + doCheck = false; meta = with stdenv.lib; { description = "Utilities to facilitate the installation of Python packages"; diff --git a/pkgs/development/python-modules/setuptools_scm/default.nix b/pkgs/development/python-modules/setuptools_scm/default.nix index a222fc9e49f..c9704196aec 100644 --- a/pkgs/development/python-modules/setuptools_scm/default.nix +++ b/pkgs/development/python-modules/setuptools_scm/default.nix @@ -8,8 +8,6 @@ buildPythonPackage rec { sha256 = "52ab47715fa0fc7d8e6cd15168d1a69ba995feb1505131c3e814eb7087b57358"; }; - buildInputs = [ pip ]; - # Seems to fail due to chroot and would cause circular dependency # with pytest doCheck = false; diff --git a/pkgs/development/python-modules/wheel/default.nix b/pkgs/development/python-modules/wheel/default.nix index d7814984060..5638e3e8be9 100644 --- a/pkgs/development/python-modules/wheel/default.nix +++ b/pkgs/development/python-modules/wheel/default.nix @@ -1,15 +1,19 @@ { lib +, setuptools +, pip , buildPythonPackage , fetchPypi , pytest , pytestcov , coverage , jsonschema +, bootstrapped-pip }: buildPythonPackage rec { pname = "wheel"; version = "0.33.4"; + format = "other"; src = fetchPypi { inherit pname version; @@ -17,14 +21,14 @@ buildPythonPackage rec { }; checkInputs = [ pytest pytestcov coverage ]; + nativeBuildInputs = [ bootstrapped-pip setuptools ]; - propagatedBuildInputs = [ jsonschema ]; - + catchConflicts = false; # No tests in archive doCheck = false; # We add this flag to ignore the copy installed by bootstrapped-pip - installFlags = [ "--ignore-installed" ]; + pipInstallFlags = [ "--ignore-installed" ]; meta = { description = "A built-package format for Python"; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 921948b3c82..750f96c6289 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -42,17 +42,14 @@ let } else ff; - buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { - flit = self.flit; - # We want Python libraries to be named like e.g. "python3.6-${name}" - inherit namePrefix; - inherit toPythonModule; + buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix { + inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}" + inherit toPythonModule; # Libraries provide modules })); - buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix { - flit = self.flit; - namePrefix = ""; - toPythonModule = x: x; # Application does not provide modules. + buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix { + namePrefix = ""; # Python applications should not have any prefix + toPythonModule = x: x; # Application does not provide modules. })); # See build-setupcfg/default.nix for documentation. @@ -110,6 +107,9 @@ in { inherit toPythonModule toPythonApplication; inherit buildSetupcfg; + inherit (callPackage ../development/interpreters/python/hooks { }) + flitBuildHook pipBuildHook pipInstallHook pytestCheckHook pythonCatchConflictsHook pythonImportsCheckHook pythonRemoveBinBytecodeHook setuptoolsBuildHook setuptoolsCheckHook wheelUnpackHook; + # helpers wrapPython = callPackage ../development/interpreters/python/wrap-python.nix {inherit python; inherit (pkgs) makeSetupHook makeWrapper; }; @@ -121,7 +121,7 @@ in { recursivePthLoader = callPackage ../development/python-modules/recursive-pth-loader { }; - setuptools = toPythonModule (callPackage ../development/python-modules/setuptools { }); + setuptools = callPackage ../development/python-modules/setuptools { }; vowpalwabbit = callPackage ../development/python-modules/vowpalwabbit { }; |