Merge pull request #254136 from figsoda/python

python310Packages.buildPythonPackage: introduce pyproject option

5 files changed, 103 insertions, 31 deletions

diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md
index e4a3a898f00..ca0513fbde8 100644
--- a/doc/languages-frameworks/python.section.md
+++ b/doc/languages-frameworks/python.section.md
@@ -101,7 +101,7 @@ The following is an example:
 buildPythonPackage rec {
   pname = "pytest";
   version = "3.3.1";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
@@ -167,12 +167,15 @@ following are specific to `buildPythonPackage`:
 * `dontWrapPythonPrograms ? false`: Skip wrapping of Python programs.
 * `permitUserSite ? false`: Skip setting the `PYTHONNOUSERSITE` environment
   variable in wrapped programs.
-* `format ? "setuptools"`: Format of the source. Valid options are
-  `"setuptools"`, `"pyproject"`, `"flit"`, `"wheel"`, and `"other"`.
-  `"setuptools"` is for when the source has a `setup.py` and `setuptools` is
-  used to build a wheel, `flit`, in case `flit` should be used to build a wheel,
-  and `wheel` in case a wheel is provided. Use `other` when a custom
-  `buildPhase` and/or `installPhase` is needed.
+* `pyproject`: Whether the pyproject format should be used. When set to `true`,
+  `pypaBuildHook` will be used, and you can add the required build dependencies
+  from `build-system.requires` to `nativeBuildInputs`. Note that the pyproject
+  format falls back to using `setuptools`, so you can use `pyproject = true`
+  even if the package only has a `setup.py`. When set to `false`, you can
+  use the existing [hooks](#setup-hooks0 or provide your own logic to build the
+  package. This can be useful for packages that don't support the pyproject
+  format. When unset, the legacy `setuptools` hooks are used for backwards
+  compatibility.
 * `makeWrapperArgs ? []`: A list of strings. Arguments to be passed to
   `makeWrapper`, which wraps generated binaries. By default, the arguments to
   `makeWrapper` set `PATH` and `PYTHONPATH` environment variables before calling
@@ -286,20 +289,25 @@ specifying an interpreter version), like this:
 python3.pkgs.buildPythonApplication rec {
   pname = "luigi";
   version = "2.7.9";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash  = "sha256-Pe229rT0aHwA98s+nTHQMEFKZPo/yw6sot8MivFDvAw=";
   };
 
+  nativeBuildInputs = [
+    python3.pkgs.setuptools
+    python3.pkgs.wheel
+  ];
+
   propagatedBuildInputs = with python3.pkgs; [
     tornado
     python-daemon
   ];
 
   meta = with lib; {
-    ...
+    # ...
   };
 }
 ```
@@ -858,18 +866,25 @@ building Python libraries is `buildPythonPackage`. Let's see how we can build th
 { lib
 , buildPythonPackage
 , fetchPypi
+, setuptools
+, wheel
 }:
 
 buildPythonPackage rec {
   pname = "toolz";
   version = "0.10.0";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
   };
 
+  nativeBuildInputs = [
+    setuptools
+    wheel
+  ];
+
   # has no tests
   doCheck = false;
 
@@ -918,13 +933,18 @@ with import <nixpkgs> {};
     my_toolz = python311.pkgs.buildPythonPackage rec {
       pname = "toolz";
       version = "0.10.0";
-      format = "setuptools";
+      pyproject = true;
 
       src = fetchPypi {
         inherit pname version;
         hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
       };
 
+      nativeBuildInputs = [
+        python311.pkgs.setuptools
+        python311.pkgs.wheel
+      ];
+
       # has no tests
       doCheck = false;
 
@@ -972,6 +992,9 @@ order to build [`datashape`](https://github.com/blaze/datashape).
 , buildPythonPackage
 , fetchPypi
 
+# build dependencies
+, setuptools, wheel
+
 # dependencies
 , numpy, multipledispatch, python-dateutil
 
@@ -982,13 +1005,18 @@ order to build [`datashape`](https://github.com/blaze/datashape).
 buildPythonPackage rec {
   pname = "datashape";
   version = "0.4.7";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash = "sha256-FLLvdm1MllKrgTGC6Gb0k0deZeVYvtCCLji/B7uhong=";
   };
 
+  nativeBuildInputs = [
+    setuptools
+    wheel
+  ];
+
   propagatedBuildInputs = [
     multipledispatch
     numpy
@@ -1023,6 +1051,8 @@ when building the bindings and are therefore added as `buildInputs`.
 { lib
 , buildPythonPackage
 , fetchPypi
+, setuptools
+, wheel
 , libxml2
 , libxslt
 }:
@@ -1030,13 +1060,18 @@ when building the bindings and are therefore added as `buildInputs`.
 buildPythonPackage rec {
   pname = "lxml";
   version = "3.4.4";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash = "sha256-s9NiusRxFydHzaNRMjjxFcvWxfi45jGb9ql6eJJyQJk=";
   };
 
+  nativeBuildInputs = [
+    setuptools
+    wheel
+  ];
+
   buildInputs = [
     libxml2
     libxslt
@@ -1067,6 +1102,10 @@ therefore we have to set `LDFLAGS` and `CFLAGS`.
 , buildPythonPackage
 , fetchPypi
 
+# build dependencies
+, setuptools
+, wheel
+
 # dependencies
 , fftw
 , fftwFloat
@@ -1078,13 +1117,18 @@ therefore we have to set `LDFLAGS` and `CFLAGS`.
 buildPythonPackage rec {
   pname = "pyFFTW";
   version = "0.9.2";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash = "sha256-9ru2r6kwhUCaskiFoaPNuJCfCVoUL01J40byvRt4kHQ=";
   };
 
+  nativeBuildInputs = [
+    setuptools
+    wheel
+  ];
+
   buildInputs = [
     fftw
     fftwFloat
@@ -1334,9 +1378,7 @@ instead of a dev dependency).
 
 Keep in mind that while the examples above are done with `requirements.txt`,
 `pythonRelaxDepsHook` works by modifying the resulting wheel file, so it should
-work in any of the formats supported by `buildPythonPackage` currently,
-with the exception of `other` (see `format` in
-[`buildPythonPackage` parameters](#buildpythonpackage-parameters) for more details).
+work with any of the existing [hooks](#setup-hooks).
 
 #### Using unittestCheckHook {#using-unittestcheckhook}
 
@@ -1461,18 +1503,26 @@ We first create a function that builds `toolz` in `~/path/to/toolz/release.nix`
 ```nix
 { lib
 , buildPythonPackage
+, fetchPypi
+, setuptools
+, wheel
 }:
 
 buildPythonPackage rec {
   pname = "toolz";
   version = "0.10.0";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
     hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";
   };
 
+  nativeBuildInputs = [
+    setuptools
+    wheel
+  ];
+
   meta = with lib; {
     changelog = "https://github.com/pytoolz/toolz/releases/tag/${version}";
     homepage = "https://github.com/pytoolz/toolz/";
diff --git a/pkgs/applications/networking/seahub/default.nix b/pkgs/applications/networking/seahub/default.nix
index 9bd90ad928c..089eabf93d6 100644
--- a/pkgs/applications/networking/seahub/default.nix
+++ b/pkgs/applications/networking/seahub/default.nix
@@ -22,7 +22,7 @@ in
 python.pkgs.buildPythonApplication rec {
   pname = "seahub";
   version = "9.0.10";
-  format = "other";
+  pyproject = false;
 
   src = fetchFromGitHub {
     owner = "haiwen";
diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix
index 8a75dbf43a7..e9c783116b6 100644
--- a/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -82,6 +82,11 @@
 # However, some packages do provide executables with extensions, and thus bytecode is generated.
 , removeBinBytecode ? true
 
+# pyproject = true <-> format = "pyproject"
+# pyproject = false <-> format = "other"
+# https://github.com/NixOS/nixpkgs/issues/253154
+, pyproject ? null
+
 # Several package formats are supported.
 # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
 # "wheel" : Install from a pre-compiled wheel.
@@ -89,7 +94,7 @@
 # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
 # "egg": Install a package from an egg.
 # "other" : Provide your own buildPhase and installPhase.
-, format ? "setuptools"
+, format ? null
 
 , meta ? {}
 
@@ -101,10 +106,23 @@
 
 , ... } @ attrs:
 
+assert (pyproject != null) -> (format == null);
+
 let
   inherit (python) stdenv;
 
-  withDistOutput = lib.elem format ["pyproject" "setuptools" "flit" "wheel"];
+  format' =
+    if pyproject != null then
+      if pyproject then
+        "pyproject"
+      else
+        "other"
+    else if format != null then
+      format
+    else
+      "setuptools";
+
+  withDistOutput = lib.elem format' ["pyproject" "setuptools" "flit" "wheel"];
 
   name_ = name;
 
@@ -177,7 +195,7 @@ let
 
   # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
   self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
-    "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format"
+    "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format"
     "disabledTestPaths" "outputs"
   ]) // {
 
@@ -202,11 +220,11 @@ let
       pythonRemoveBinBytecodeHook
     ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [
       unzip
-    ] ++ lib.optionals (format == "setuptools") [
+    ] ++ lib.optionals (format' == "setuptools") [
       setuptoolsBuildHook
-    ] ++ lib.optionals (format == "flit") [
+    ] ++ lib.optionals (format' == "flit") [
       flitBuildHook
-    ] ++ lib.optionals (format == "pyproject") [(
+    ] ++ lib.optionals (format' == "pyproject") [(
       if isBootstrapPackage then
         pypaBuildHook.override {
           inherit (python.pythonForBuild.pkgs.bootstrap) build;
@@ -214,11 +232,11 @@ let
         }
       else
         pypaBuildHook
-    )] ++ lib.optionals (format == "wheel") [
+    )] ++ lib.optionals (format' == "wheel") [
       wheelUnpackHook
-    ] ++ lib.optionals (format == "egg") [
+    ] ++ lib.optionals (format' == "egg") [
       eggUnpackHook eggBuildHook eggInstallHook
-    ] ++ lib.optionals (format != "other") [(
+    ] ++ lib.optionals (format' != "other") [(
       if isBootstrapInstallPackage then
         pypaInstallHook.override {
           inherit (python.pythonForBuild.pkgs.bootstrap) installer;
@@ -252,7 +270,7 @@ let
     doCheck = false;
     doInstallCheck = attrs.doCheck or true;
     nativeInstallCheckInputs = [
-    ] ++ lib.optionals (format == "setuptools") [
+    ] ++ lib.optionals (format' == "setuptools") [
       # Longer-term we should get rid of this and require
       # users of this function to set the `installCheckPhase` or
       # pass in a hook that sets it.
diff --git a/pkgs/development/tools/language-servers/ruff-lsp/default.nix b/pkgs/development/tools/language-servers/ruff-lsp/default.nix
index 4f4304017c4..abd9bbb52c9 100644
--- a/pkgs/development/tools/language-servers/ruff-lsp/default.nix
+++ b/pkgs/development/tools/language-servers/ruff-lsp/default.nix
@@ -16,7 +16,7 @@
 buildPythonPackage rec {
   pname = "ruff-lsp";
   version = "0.0.39";
-  format = "pyproject";
+  pyproject = true;
   disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
diff --git a/pkgs/tools/package-management/nix-update/default.nix b/pkgs/tools/package-management/nix-update/default.nix
index 2dba8c0c135..c0030fcf8a6 100644
--- a/pkgs/tools/package-management/nix-update/default.nix
+++ b/pkgs/tools/package-management/nix-update/default.nix
@@ -10,7 +10,7 @@
 python3.pkgs.buildPythonApplication rec {
   pname = "nix-update";
   version = "0.19.3";
-  format = "setuptools";
+  pyproject = true;
 
   src = fetchFromGitHub {
     owner = "Mic92";
@@ -19,6 +19,10 @@ python3.pkgs.buildPythonApplication rec {
     hash = "sha256-+WD+SV/L3TvksWBIg6jk+T0dUTNdp4VKONzdzVT+pac=";
   };
 
+  nativeBuildInputs = [
+    python3.pkgs.setuptools
+  ];
+
   makeWrapperArgs = [
     "--prefix" "PATH" ":" (lib.makeBinPath [ nix nix-prefetch-git nixpkgs-fmt nixpkgs-review ])
   ];