diff options
Diffstat (limited to 'pkgs/development/tools/poetry2nix/poetry2nix')
7 files changed, 278 insertions, 166 deletions
diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index 967f0d666cb..b6d53387552 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -14,22 +14,16 @@ let defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; }); mkEvalPep508 = import ./pep508.nix { - inherit lib; + inherit lib poetryLib; stdenv = pkgs.stdenv; }; getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn; - getAttrDefault = attribute: set: default: ( - if builtins.hasAttr attribute set - then builtins.getAttr attribute set - else default - ); - # Map SPDX identifiers to license names spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses))); # Get license by id falling back to input string - getLicenseBySpdxId = spdxId: getAttrDefault spdxId spdxLicenses spdxId; + getLicenseBySpdxId = spdxId: spdxLicenses.${spdxId} or spdxId; # # Returns an attrset { python, poetryPackages } for the given lockfile @@ -65,7 +59,7 @@ let # closure as python can only ever have one version of a dependency baseOverlay = self: super: let - getDep = depName: if builtins.hasAttr depName self then self."${depName}" else throw "foo"; + getDep = depName: self.${depName}; lockPkgs = builtins.listToAttrs ( builtins.map ( @@ -74,7 +68,7 @@ let value = self.mkPoetryDep ( pkgMeta // { inherit pwd; - source = getAttrDefault "source" pkgMeta null; + source = pkgMeta.source or null; files = lockFiles.${name}; pythonPackages = self; } @@ -159,12 +153,12 @@ let passedAttrs = builtins.removeAttrs attrs specialAttrs; getDeps = depAttr: let - deps = getAttrDefault depAttr pyProject.tool.poetry {}; + deps = pyProject.tool.poetry.${depAttr} or {}; depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps); in builtins.map (dep: py.pkgs."${dep}") depAttrs; - getInputs = attr: getAttrDefault attr attrs []; + getInputs = attr: attrs.${attr} or []; mkInput = attr: extraInputs: getInputs attr ++ extraInputs; buildSystemPkgs = poetryLib.getBuildSystemPkgs { @@ -189,7 +183,7 @@ let python = py; }; - postPatch = (getAttrDefault "postPatch" passedAttrs "") + '' + postPatch = (passedAttrs.postPatch or "") + '' # Tell poetry not to resolve the path dependencies. Any version is # fine ! yj -tj < pyproject.toml | python ${./pyproject-without-path.py} > pyproject.json @@ -199,7 +193,7 @@ let meta = meta // { inherit (pyProject.tool.poetry) description homepage; - license = getLicenseBySpdxId (getAttrDefault "license" pyProject.tool.poetry "unknown"); + license = getLicenseBySpdxId (pyProject.tool.poetry.license or "unknown"); }; } @@ -247,6 +241,7 @@ in overrideOverlay = fn: self: super: let defaultSet = defaultPoetryOverrides self super; customSet = fn self super; - in defaultSet // customSet; + in + defaultSet // customSet; }; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json b/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json index 2cce8e2ea08..33052efe296 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json +++ b/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json @@ -1,4 +1,5 @@ [ + "egg", "tar", "tar.bz2", "tar.gz", @@ -11,4 +12,4 @@ "txz", "whl", "zip" -] \ No newline at end of file +] diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix index 9ec76defb7d..68d854f2648 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -1,6 +1,12 @@ { lib, pkgs }: let - inherit (import ./semver.nix { inherit lib; }) satisfiesSemver; + inherit (import ./semver.nix { inherit lib ireplace; }) satisfiesSemver; + inherit (builtins) genList length; + + # Replace a list entry at defined index with set value + ireplace = idx: value: list: ( + genList (i: if i == idx then value else (builtins.elemAt list i)) (length list) + ); # Returns true if pythonVersion matches with the expression in pythonVersions isCompatible = pythonVersion: pythonVersions: @@ -24,7 +30,27 @@ let in (builtins.foldl' combine initial tokens).state; - readTOML = path: builtins.fromTOML (builtins.readFile path); + fromTOML = builtins.fromTOML or + ( + toml: builtins.fromJSON ( + builtins.readFile ( + pkgs.runCommand "from-toml" + { + inherit toml; + allowSubstitutes = false; + preferLocalBuild = true; + } + '' + ${pkgs.remarshal}/bin/remarshal \ + -if toml \ + -i <(echo "$toml") \ + -of json \ + -o $out + '' + ) + ) + ); + readTOML = path: fromTOML (builtins.readFile path); # # Returns the appropriate manylinux dependencies and string representation for the file specified diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index 3631cbd228e..95543ca7359 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -16,85 +16,112 @@ , pwd , supportedExtensions ? lib.importJSON ./extensions.json , ... -}: let - - inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi; - - inherit (import ./pep425.nix { - inherit lib python; - inherit (pkgs) stdenv; - }) selectWheel - ; - - fileCandidates = let - supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")"); - matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null; - hasSupportedExtension = fname: builtins.match supportedRegex fname != null; - in - builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file) files; - - toPath = s: pwd + "/${s}"; - - isSource = source != null; - isGit = isSource && source.type == "git"; - isLocal = isSource && source.type == "directory"; - - localDepPath = toPath source.url; - pyProject = poetryLib.readTOML (localDepPath + "/pyproject.toml"); - - buildSystemPkgs = poetryLib.getBuildSystemPkgs { - inherit pythonPackages pyProject; - }; - - fileInfo = let - isBdist = f: lib.strings.hasSuffix "whl" f.file; - isSdist = f: ! isBdist f; - binaryDist = selectWheel fileCandidates; - sourceDist = builtins.filter isSdist fileCandidates; - lockFileEntry = if (builtins.length sourceDist) > 0 then builtins.head sourceDist else builtins.head binaryDist; - in - rec { - inherit (lockFileEntry) file hash; - name = file; - format = if lib.strings.hasSuffix ".whl" name then "wheel" else "setuptools"; - kind = if format == "setuptools" then "source" else (builtins.elemAt (lib.strings.splitString "-" name) 2); - }; - -in -buildPythonPackage { - pname = name; - version = version; - - doCheck = false; # We never get development deps - dontStrip = true; - format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; - - nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []; - buildInputs = if !isSource then (getManyLinuxDeps fileInfo.name).pkg else []; - - propagatedBuildInputs = +}: + +pythonPackages.callPackage ( + { preferWheel ? false + }: + let - # Some dependencies like django gets the attribute name django - # but dependencies try to access Django - deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies); + + inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi; + + inherit (import ./pep425.nix { + inherit lib python; + inherit (pkgs) stdenv; + }) selectWheel + ; + + fileCandidates = let + supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")"); + matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null; + hasSupportedExtension = fname: builtins.match supportedRegex fname != null; + isCompatibleEgg = fname: ! lib.strings.hasSuffix ".egg" fname || lib.strings.hasSuffix "py${python.pythonVersion}.egg" fname; + in + builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file && isCompatibleEgg f.file) files; + + toPath = s: pwd + "/${s}"; + + isSource = source != null; + isGit = isSource && source.type == "git"; + isLocal = isSource && source.type == "directory"; + + localDepPath = toPath source.url; + pyProject = poetryLib.readTOML (localDepPath + "/pyproject.toml"); + + buildSystemPkgs = poetryLib.getBuildSystemPkgs { + inherit pythonPackages pyProject; + }; + + fileInfo = let + isBdist = f: lib.strings.hasSuffix "whl" f.file; + isSdist = f: ! isBdist f && ! isEgg f; + isEgg = f: lib.strings.hasSuffix ".egg" f.file; + + binaryDist = selectWheel fileCandidates; + sourceDist = builtins.filter isSdist fileCandidates; + eggs = builtins.filter isEgg fileCandidates; + + entries = (if preferWheel then binaryDist ++ sourceDist else sourceDist ++ binaryDist) ++ eggs; + + lockFileEntry = builtins.head entries; + + _isEgg = isEgg lockFileEntry; + + in + rec { + inherit (lockFileEntry) file hash; + name = file; + format = + if _isEgg then "egg" + else if lib.strings.hasSuffix ".whl" name then "wheel" + else "setuptools"; + kind = + if _isEgg then python.pythonVersion + else if format == "setuptools" then "source" + else (builtins.elemAt (lib.strings.splitString "-" name) 2); + }; + + baseBuildInputs = lib.optional (name != "setuptools_scm" && name != "setuptools-scm") pythonPackages.setuptools_scm; + in - (builtins.map (n: pythonPackages.${n}) deps) ++ (if isLocal then buildSystemPkgs else []); - - meta = { - broken = ! isCompatible python.version python-versions; - license = []; - }; - - # We need to retrieve kind from the interpreter and the filename of the package - # Interpreters should declare what wheel types they're compatible with (python type + ABI) - # Here we can then choose a file based on that info. - src = if isGit then ( - builtins.fetchGit { - inherit (source) url; - rev = source.reference; - } - ) else if isLocal then (localDepPath) else fetchFromPypi { - pname = name; - inherit (fileInfo) file hash kind; - }; -} + + buildPythonPackage { + pname = name; + version = version; + + doCheck = false; # We never get development deps + dontStrip = true; + format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; + + nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []; + buildInputs = baseBuildInputs ++ (if !isSource then (getManyLinuxDeps fileInfo.name).pkg else []); + + propagatedBuildInputs = + let + # Some dependencies like django gets the attribute name django + # but dependencies try to access Django + deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies); + in + (builtins.map (n: pythonPackages.${n}) deps) ++ (if isLocal then buildSystemPkgs else []); + + meta = { + broken = ! isCompatible python.version python-versions; + license = []; + }; + + # We need to retrieve kind from the interpreter and the filename of the package + # Interpreters should declare what wheel types they're compatible with (python type + ABI) + # Here we can then choose a file based on that info. + src = if isGit then ( + builtins.fetchGit { + inherit (source) url; + rev = source.reference; + } + ) else if isLocal then (localDepPath) else fetchFromPypi { + pname = name; + inherit (fileInfo) file hash kind; + }; + } + +) {} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 84d77cef3b5..48b8ff9859b 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -5,32 +5,7 @@ self: super: -let - - addSetupTools = drv: if drv == null then null else drv.overrideAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.setuptools_scm - ]; - } - ); - - getAttrDefault = attribute: set: default: - if builtins.hasAttr attribute set - then builtins.getAttr attribute set - else default; - -in { - - asciimatics = super.asciimatics.overrideAttrs ( - old: { - buildInputs = old.buildInputs ++ [ - self.setuptools_scm - ]; - } - ); - av = super.av.overrideAttrs ( old: { nativeBuildInputs = old.nativeBuildInputs ++ [ @@ -60,10 +35,6 @@ in } ); - configparser = addSetupTools super.configparser; - - cbor2 = addSetupTools super.cbor2; - cryptography = super.cryptography.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ pkgs.openssl ]; @@ -73,7 +44,7 @@ in django = ( super.django.overrideAttrs ( old: { - propagatedNativeBuildInputs = (getAttrDefault "propagatedNativeBuildInputs" old []) + propagatedNativeBuildInputs = (old.propagatedNativeBuildInputs or []) ++ [ pkgs.gettext ]; } ) @@ -85,7 +56,7 @@ in if ! test -e LICENSE; then touch LICENSE fi - '' + (getAttrDefault "configurePhase" old ""); + '' + (old.configurePhase or ""); } ); @@ -106,22 +77,13 @@ in } ); - hypothesis = addSetupTools super.hypothesis; - - importlib-metadata = addSetupTools super.importlib-metadata; - - inflect = super.inflect.overrideAttrs ( + # importlib-metadata has an incomplete dependency specification + importlib-metadata = super.importlib-metadata.overrideAttrs ( old: { - buildInputs = old.buildInputs ++ [ - self.setuptools_scm - ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ lib.optional self.python.isPy2 self.pathlib2; } ); - jsonschema = addSetupTools super.jsonschema; - - keyring = addSetupTools super.keyring; - lap = super.lap.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ [ @@ -191,6 +153,11 @@ in } ); + # Calls Cargo at build time for source builds and is really tricky to package + maturin = super.maturin.override { + preferWheel = true; + }; + mccabe = super.mccabe.overrideAttrs ( old: { postPatch = '' @@ -243,7 +210,7 @@ in in { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; - buildInputs = old.buildInputs ++ [ blas ]; + buildInputs = old.buildInputs ++ [ blas self.cython ]; enableParallelBuilding = true; preBuild = '' ln -s ${cfg} site.cfg @@ -262,8 +229,6 @@ in } ); - pluggy = addSetupTools super.pluggy; - psycopg2 = super.psycopg2.overrideAttrs ( old: { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; @@ -276,8 +241,6 @@ in } ); - py = addSetupTools super.py; - pyarrow = super.pyarrow.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ @@ -334,16 +297,96 @@ in } ); - pytest = addSetupTools super.pytest; + pyqt5 = super.pyqt5.overridePythonAttrs ( + old: { + format = "other"; + + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + pkgs.qt5.qmake + pkgs.xorg.lndir + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + pkgs.qt5.qtwebchannel + # self.pyqt5-sip + self.sip + ]; + + buildInputs = old.buildInputs ++ [ + pkgs.dbus + pkgs.qt5.qtbase + pkgs.qt5.qtsvg + pkgs.qt5.qtdeclarative + self.sip + ]; + + # Fix dbus mainloop + inherit (pkgs.python3.pkgs.pyqt5) patches; + + configurePhase = '' + runHook preConfigure + + export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} + + mkdir -p $out/${self.python.sitePackages}/dbus/mainloop + ${self.python.executable} configure.py -w \ + --confirm-license \ + --no-qml-plugin \ + --bindir=$out/bin \ + --destdir=$out/${self.python.sitePackages} \ + --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ + --sipdir=$out/share/sip/PyQt5 \ + --designer-plugindir=$out/plugins/designer + + runHook postConfigure + ''; + + postInstall = '' + ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ + for i in $out/bin/*; do + wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" + done + + # # Let's make it a namespace package + # cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py + # from pkgutil import extend_path + # __path__ = extend_path(__path__, __name__) + # EOF + ''; + + installCheckPhase = let + modules = [ + "PyQt5" + "PyQt5.QtCore" + "PyQt5.QtQml" + "PyQt5.QtWidgets" + "PyQt5.QtGui" + ]; + imports = lib.concatMapStrings (module: "import ${module};") modules; + in + '' + echo "Checking whether modules can be imported..." + ${self.python.interpreter} -c "${imports}" + ''; + + doCheck = true; - pytest-mock = addSetupTools super.pytest-mock; + enableParallelBuilding = true; + } + ); - python-dateutil = addSetupTools super.python-dateutil; + pytest-datadir = super.pytest-datadir.overrideAttrs ( + old: { + postInstall = '' + rm -f $out/LICENSE + ''; + } + ); python-prctl = super.python-prctl.overrideAttrs ( old: { buildInputs = old.buildInputs ++ [ - self.setuptools_scm pkgs.libcap ]; } @@ -380,8 +423,6 @@ in } ); - six = addSetupTools super.six; - urwidtrees = super.urwidtrees.overrideAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ [ @@ -390,7 +431,15 @@ in } ); - # TODO: Figure out getting rid of this hack + vose-alias-method = super.pytest-datadir.overrideAttrs ( + old: { + postInstall = '' + rm -f $out/LICENSE + ''; + } + ); + + # Stop infinite recursion by using bootstrapped pkg from nixpkgs wheel = ( pkgs.python3.pkgs.override { python = self.python; @@ -401,5 +450,4 @@ in } ); - zipp = addSetupTools super.zipp; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix index bf1893931cd..93a395326eb 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix @@ -1,6 +1,7 @@ -{ lib, stdenv }: python: +{ lib, stdenv, poetryLib }: python: let + inherit (poetryLib) ireplace; # Like builtins.substring but with stop being offset instead of length substr = start: stop: s: builtins.substring start (stop - start) s; @@ -142,7 +143,6 @@ let else builtins.fromJSON v ); hasElem = needle: haystack: builtins.elem needle (builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " haystack)); - # TODO: Implement all operators op = { "<=" = x: y: (unmarshal x) <= (unmarshal y); "<" = x: y: (unmarshal x) < (unmarshal y); @@ -150,8 +150,16 @@ let "==" = x: y: x == y; ">=" = x: y: (unmarshal x) >= (unmarshal y); ">" = x: y: (unmarshal x) > (unmarshal y); - "~=" = null; - "===" = null; + "~=" = v: c: let + parts = builtins.splitVersion c; + pruned = lib.take ((builtins.length parts) - 1) parts; + upper = builtins.toString ( + (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 + ); + upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned); + in + op.">=" v c && op."<" v upperConstraint; + "===" = x: y: x == y; "in" = x: y: let values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y)); in diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix index 620bb25ad97..784589a4ca4 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix @@ -1,14 +1,8 @@ -{ lib }: +{ lib, ireplace }: let inherit (builtins) elemAt match; - # Replace a list entry at defined index with set value - ireplace = idx: value: list: let - inherit (builtins) genList length; - in - genList (i: if i == idx then value else (elemAt list i)) (length list); - operators = let matchWildCard = s: match "([^\*])(\.[\*])" s; mkComparison = ret: version: v: builtins.compareVersions version v == ret; @@ -37,10 +31,23 @@ let ">=" = v: c: operators."==" v c || operators.">" v c; "<=" = v: c: operators."==" v c || operators."<" v c; # Semver specific operators - "~" = mkIdxComparison 1; # + "~" = mkIdxComparison 1; "^" = mkIdxComparison 0; + "~=" = v: c: let + # Prune constraint + parts = builtins.splitVersion c; + pruned = lib.take ((builtins.length parts) - 1) parts; + upper = builtins.toString ( + (lib.toInt (builtins.elemAt pruned (builtins.length pruned - 1))) + 1 + ); + upperConstraint = builtins.concatStringsSep "." (ireplace (builtins.length pruned - 1) upper pruned); + in + operators.">=" v c && operators."<" v upperConstraint; # Infix operators "-" = version: v: operators.">=" version v.vl && operators."<=" version v.vu; + # Arbitrary equality clause, just run simple comparison + "===" = v: c: v == c; + # }; re = { |