diff options
Diffstat (limited to 'pkgs/development/tools/poetry2nix')
13 files changed, 1459 insertions, 0 deletions
diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/README.md b/pkgs/development/tools/poetry2nix/poetry2nix/README.md new file mode 100644 index 00000000000..ac4861534f5 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/README.md @@ -0,0 +1,6 @@ +Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix + +The update procedure is as-follows: +1. Send your change to the upstream poetry2nix repository +2. Get it approved with tests passing +3. Run the update script in pkgs/development/tools/poetry2nix diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix b/pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix new file mode 100755 index 00000000000..95576b987f5 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix @@ -0,0 +1,98 @@ +#!/usr/bin/env python +from concurrent.futures import ThreadPoolExecutor +import subprocess +import textwrap +import argparse +import toml +import json +import sys + + +argparser = argparse.ArgumentParser(description="Generate overrides for git hashes",) +argparser.add_argument( + "--lock", default="poetry.lock", help="Path to input poetry.lock", +) +argparser.add_argument( + "--out", default="poetry-git-overlay.nix", help="Output file", +) + + +def fetch_git(pkg): + return ( + pkg["name"], + subprocess.run( + [ + "nix-prefetch-git", + "--fetch-submodules", + "--url", + pkg["source"]["url"], + "--rev", + pkg["source"]["reference"], + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ), + ) + + +def indent(expr, spaces=2): + i = " " * spaces + return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")]) + + +if __name__ == "__main__": + args = argparser.parse_args() + + with open(args.lock) as lockf: + lock = toml.load(lockf) + + pkgs = [] + for pkg in lock["package"]: + if "source" in pkg: + pkgs.append(pkg) + + with ThreadPoolExecutor() as e: + futures = [] + + for pkg in pkgs: + futures.append(e.submit(fetch_git, pkg)) + + lines = [ + "{ pkgs }:", + "self: super: {", + ] + + for f in futures: + drv_name, p = f.result() + if p.returncode != 0: + sys.stderr.buffer.write(p.stderr) + sys.stderr.buffer.flush() + exit(p.returncode) + + meta = json.loads(p.stdout.decode()) + lines.append( + indent( + textwrap.dedent( + """ + %s = super.%s.overrideAttrs ( + _: { + src = pkgs.fetchgit { + url = "%s"; + rev = "%s"; + sha256 = "%s"; + }; + } + );""" + % (drv_name, drv_name, meta["url"], meta["rev"], meta["sha256"]) + ) + ) + ) + + lines.extend(["", "}", ""]) + + expr = "\n".join(lines) + + with open(args.out, "w") as f: + f.write(expr) + + print(f"Wrote {args.out}") diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix new file mode 100644 index 00000000000..82759ce7174 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/cli.nix @@ -0,0 +1,52 @@ +{ pkgs ? import <nixpkgs> {} +, lib ? pkgs.lib +, version +}: + +let + inherit (pkgs) python3; + +in +pkgs.stdenv.mkDerivation { + pname = "poetry2nix"; + inherit version; + + buildInputs = [ + (python3.withPackages (ps: [ ps.toml ])) + ]; + + nativeBuildInputs = [ + pkgs.makeWrapper + ]; + + src = ./bin; + + dontConfigure = true; + + buildPhase = '' + runHook preBuild + ${python3.pkgs.black}/bin/black --quiet --check poetry2nix + patchShebangs poetry2nix + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + mv poetry2nix $out/bin + + wrapProgram $out/bin/poetry2nix --prefix PATH ":" ${lib.makeBinPath [ + pkgs.nix-prefetch-git + ]} + + runHook postInstall + ''; + + meta = { + homepage = "https://github.com/nix-community/poetry2nix"; + description = "CLI to supplement sha256 hashes for git dependencies"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.adisbladis ]; + }; + +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix new file mode 100644 index 00000000000..967f0d666cb --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -0,0 +1,252 @@ +{ pkgs ? import <nixpkgs> {} +, lib ? pkgs.lib +, poetry ? null +, poetryLib ? import ./lib.nix { inherit lib pkgs; } +}: + +let + inherit (poetryLib) isCompatible readTOML; + + # Poetry2nix version + version = "1.1.0"; + + /* The default list of poetry2nix override overlays */ + defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; }); + + mkEvalPep508 = import ./pep508.nix { + inherit lib; + stdenv = pkgs.stdenv; + }; + + getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn; + + getAttrDefault = attribute: set: default: ( + if builtins.hasAttr attribute set + then builtins.getAttr attribute set + else default + ); + + # Map SPDX identifiers to license names + spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses))); + # Get license by id falling back to input string + getLicenseBySpdxId = spdxId: getAttrDefault spdxId spdxLicenses spdxId; + + # + # Returns an attrset { python, poetryPackages } for the given lockfile + # + mkPoetryPython = + { poetrylock + , poetryPkg + , overrides ? [ defaultPoetryOverrides ] + , meta ? {} + , python ? pkgs.python3 + , pwd ? null + }@attrs: let + lockData = readTOML poetrylock; + lockFiles = lib.getAttrFromPath [ "metadata" "files" ] lockData; + + specialAttrs = [ "poetrylock" "overrides" ]; + passedAttrs = builtins.removeAttrs attrs specialAttrs; + + evalPep508 = mkEvalPep508 python; + + # Filter packages by their PEP508 markers + partitions = let + supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true; + in + lib.partition supportsPythonVersion lockData.package; + + compatible = partitions.right; + incompatible = partitions.wrong; + + # Create an overriden version of pythonPackages + # + # We need to avoid mixing multiple versions of pythonPackages in the same + # closure as python can only ever have one version of a dependency + baseOverlay = self: super: + let + getDep = depName: if builtins.hasAttr depName self then self."${depName}" else throw "foo"; + + lockPkgs = builtins.listToAttrs ( + builtins.map ( + pkgMeta: rec { + name = pkgMeta.name; + value = self.mkPoetryDep ( + pkgMeta // { + inherit pwd; + source = getAttrDefault "source" pkgMeta null; + files = lockFiles.${name}; + pythonPackages = self; + } + ); + } + ) compatible + ); + in + lockPkgs; + + overlays = builtins.map getFunctorFn ( + [ + ( + self: super: { + mkPoetryDep = self.callPackage ./mk-poetry-dep.nix { + inherit pkgs lib python poetryLib; + }; + poetry = poetryPkg; + } + ) + # Null out any filtered packages, we don't want python.pkgs from nixpkgs + (self: super: builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible)) + # Create poetry2nix layer + baseOverlay + ] ++ # User provided overrides + overrides + ); + + packageOverrides = lib.foldr lib.composeExtensions (self: super: {}) overlays; + + py = python.override { inherit packageOverrides; self = py; }; + in + { + python = py; + poetryPackages = map (pkg: py.pkgs.${pkg.name}) compatible; + }; + + /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file. + + Example: + poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; } + */ + mkPoetryEnv = + { poetrylock + , overrides ? [ defaultPoetryOverrides ] + , meta ? {} + , pwd ? null + , python ? pkgs.python3 + }: + let + poetryPkg = poetry.override { inherit python; }; + py = mkPoetryPython ( + { + inherit poetryPkg poetrylock overrides meta python pwd; + } + ); + in + py.python.withPackages (_: py.poetryPackages); + + /* Creates a Python application from pyproject.toml and poetry.lock */ + mkPoetryApplication = + { src + , pyproject + , poetrylock + , overrides ? [ defaultPoetryOverrides ] + , meta ? {} + , python ? pkgs.python3 + , pwd ? null + , ... + }@attrs: let + poetryPkg = poetry.override { inherit python; }; + + py = ( + mkPoetryPython { + inherit poetryPkg poetrylock overrides meta python pwd; + } + ).python; + + pyProject = readTOML pyproject; + + specialAttrs = [ "pyproject" "poetrylock" "overrides" ]; + passedAttrs = builtins.removeAttrs attrs specialAttrs; + + getDeps = depAttr: let + deps = getAttrDefault depAttr pyProject.tool.poetry {}; + depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps); + in + builtins.map (dep: py.pkgs."${dep}") depAttrs; + + getInputs = attr: getAttrDefault attr attrs []; + mkInput = attr: extraInputs: getInputs attr ++ extraInputs; + + buildSystemPkgs = poetryLib.getBuildSystemPkgs { + inherit pyProject; + pythonPackages = py.pkgs; + }; + + in + py.pkgs.buildPythonApplication ( + passedAttrs // { + pname = pyProject.tool.poetry.name; + version = pyProject.tool.poetry.version; + + format = "pyproject"; + + nativeBuildInputs = [ pkgs.yj ]; + buildInputs = mkInput "buildInputs" buildSystemPkgs; + propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]); + checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies"); + + passthru = { + python = py; + }; + + postPatch = (getAttrDefault "postPatch" passedAttrs "") + '' + # Tell poetry not to resolve the path dependencies. Any version is + # fine ! + yj -tj < pyproject.toml | python ${./pyproject-without-path.py} > pyproject.json + yj -jt < pyproject.json > pyproject.toml + rm pyproject.json + ''; + + meta = meta // { + inherit (pyProject.tool.poetry) description homepage; + license = getLicenseBySpdxId (getAttrDefault "license" pyProject.tool.poetry "unknown"); + }; + + } + ); + + /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies */ + cli = import ./cli.nix { inherit pkgs lib version; }; + + /* Poetry2nix documentation */ + doc = pkgs.stdenv.mkDerivation { + pname = "poetry2nix-docs"; + inherit version; + + src = pkgs.runCommandNoCC "poetry2nix-docs-src" {} '' + mkdir -p $out + cp ${./default.nix} $out/default.nix + ''; + + buildInputs = [ + pkgs.nixdoc + ]; + + buildPhase = '' + nixdoc --category poetry2nix --description "Poetry2nix functions" --file ./default.nix > poetry2nix.xml + ''; + + installPhase = '' + mkdir -p $out + cp poetry2nix.xml $out/ + ''; + + }; + +in +{ + inherit mkPoetryEnv mkPoetryApplication cli doc; + + /* + The default list of poetry2nix override overlays + + Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function + */ + defaultPoetryOverrides = { + __functor = defaultPoetryOverrides; + overrideOverlay = fn: self: super: let + defaultSet = defaultPoetryOverrides self super; + customSet = fn self super; + in defaultSet // customSet; + }; +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json b/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json new file mode 100644 index 00000000000..2cce8e2ea08 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/extensions.json @@ -0,0 +1,14 @@ +[ + "tar", + "tar.bz2", + "tar.gz", + "tar.lz", + "tar.lzma", + "tar.xz", + "tbz", + "tgz", + "tlz", + "txz", + "whl", + "zip" +] \ No newline at end of file diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix new file mode 100644 index 00000000000..9ec76defb7d --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -0,0 +1,79 @@ +{ lib, pkgs }: +let + inherit (import ./semver.nix { inherit lib; }) satisfiesSemver; + + # Returns true if pythonVersion matches with the expression in pythonVersions + isCompatible = pythonVersion: pythonVersions: + let + operators = { + "||" = cond1: cond2: cond1 || cond2; + "," = cond1: cond2: cond1 && cond2; # , means && + }; + # split string at "," and "||" + tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions); + combine = acc: v: + let + isOperator = builtins.typeOf v == "list"; + operator = if isOperator then (builtins.elemAt v 0) else acc.operator; + in + if isOperator then (acc // { inherit operator; }) else { + inherit operator; + state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v); + }; + initial = { operator = ","; state = true; }; + in + (builtins.foldl' combine initial tokens).state; + + readTOML = path: builtins.fromTOML (builtins.readFile path); + + # + # Returns the appropriate manylinux dependencies and string representation for the file specified + # + getManyLinuxDeps = f: + let + ml = pkgs.pythonManylinuxPackages; + in + if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; } + else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; } + else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; } + else { pkg = []; str = null; }; + + # Fetch the artifacts from the PyPI index. Since we get all + # info we need from the lock file we don't use nixpkgs' fetchPyPi + # as it modifies casing while not providing anything we don't already + # have. + # + # Args: + # pname: package name + # file: filename including extension + # hash: SRI hash + # kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention + fetchFromPypi = lib.makeOverridable ( + { pname, file, hash, kind }: + pkgs.fetchurl { + url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}"; + inherit hash; + } + ); + + getBuildSystemPkgs = + { pythonPackages + , pyProject + }: let + buildSystem = lib.getAttrFromPath [ "build-system" "build-backend" ] pyProject; + drvAttr = builtins.elemAt (builtins.split "\\.|:" buildSystem) 0; + in + if buildSystem == "" then [] else ( + [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ] + ); + +in +{ + inherit + fetchFromPypi + getManyLinuxDeps + isCompatible + readTOML + getBuildSystemPkgs + ; +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix new file mode 100644 index 00000000000..3631cbd228e --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -0,0 +1,100 @@ +{ autoPatchelfHook +, pkgs +, lib +, python +, buildPythonPackage +, pythonPackages +, poetryLib +}: +{ name +, version +, files +, source +, dependencies ? {} +, pythonPackages +, python-versions +, pwd +, supportedExtensions ? lib.importJSON ./extensions.json +, ... +}: let + + inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi; + + inherit (import ./pep425.nix { + inherit lib python; + inherit (pkgs) stdenv; + }) selectWheel + ; + + fileCandidates = let + supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")"); + matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null; + hasSupportedExtension = fname: builtins.match supportedRegex fname != null; + in + builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file) files; + + toPath = s: pwd + "/${s}"; + + isSource = source != null; + isGit = isSource && source.type == "git"; + isLocal = isSource && source.type == "directory"; + + localDepPath = toPath source.url; + pyProject = poetryLib.readTOML (localDepPath + "/pyproject.toml"); + + buildSystemPkgs = poetryLib.getBuildSystemPkgs { + inherit pythonPackages pyProject; + }; + + fileInfo = let + isBdist = f: lib.strings.hasSuffix "whl" f.file; + isSdist = f: ! isBdist f; + binaryDist = selectWheel fileCandidates; + sourceDist = builtins.filter isSdist fileCandidates; + lockFileEntry = if (builtins.length sourceDist) > 0 then builtins.head sourceDist else builtins.head binaryDist; + in + rec { + inherit (lockFileEntry) file hash; + name = file; + format = if lib.strings.hasSuffix ".whl" name then "wheel" else "setuptools"; + kind = if format == "setuptools" then "source" else (builtins.elemAt (lib.strings.splitString "-" name) 2); + }; + +in +buildPythonPackage { + pname = name; + version = version; + + doCheck = false; # We never get development deps + dontStrip = true; + format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format; + + nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else []; + buildInputs = if !isSource then (getManyLinuxDeps fileInfo.name).pkg else []; + + propagatedBuildInputs = + let + # Some dependencies like django gets the attribute name django + # but dependencies try to access Django + deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies); + in + (builtins.map (n: pythonPackages.${n}) deps) ++ (if isLocal then buildSystemPkgs else []); + + meta = { + broken = ! isCompatible python.version python-versions; + license = []; + }; + + # We need to retrieve kind from the interpreter and the filename of the package + # Interpreters should declare what wheel types they're compatible with (python type + ABI) + # Here we can then choose a file based on that info. + src = if isGit then ( + builtins.fetchGit { + inherit (source) url; + rev = source.reference; + } + ) else if isLocal then (localDepPath) else fetchFromPypi { + pname = name; + inherit (fileInfo) file hash kind; + }; +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix new file mode 100644 index 00000000000..84d77cef3b5 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -0,0 +1,405 @@ +{ pkgs ? import <nixpkgs> {} +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +}: + +self: super: + +let + + addSetupTools = drv: if drv == null then null else drv.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.setuptools_scm + ]; + } + ); + + getAttrDefault = attribute: set: default: + if builtins.hasAttr attribute set + then builtins.getAttr attribute set + else default; + +in +{ + + asciimatics = super.asciimatics.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.setuptools_scm + ]; + } + ); + + av = super.av.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + ]; + buildInputs = old.buildInputs ++ [ pkgs.ffmpeg_4 ]; + } + ); + + bcrypt = super.bcrypt.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libffi ]; + } + ); + + cffi = super.cffi.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libffi ]; + } + ); + + cftime = super.cftime.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + ]; + } + ); + + configparser = addSetupTools super.configparser; + + cbor2 = addSetupTools super.cbor2; + + cryptography = super.cryptography.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.openssl ]; + } + ); + + django = ( + super.django.overrideAttrs ( + old: { + propagatedNativeBuildInputs = (getAttrDefault "propagatedNativeBuildInputs" old []) + ++ [ pkgs.gettext ]; + } + ) + ); + + django-bakery = super.django-bakery.overrideAttrs ( + old: { + configurePhase = '' + if ! test -e LICENSE; then + touch LICENSE + fi + '' + (getAttrDefault "configurePhase" old ""); + } + ); + + # Environment markers are not always included (depending on how a dep was defined) + enum34 = if self.pythonAtLeast "3.4" then null else super.enum34; + + grandalf = super.grandalf.overrideAttrs ( + old: { + postPatch = '' + substituteInPlace setup.py --replace "setup_requires=['pytest-runner',]," "setup_requires=[]," || true + ''; + } + ); + + horovod = super.horovod.overrideAttrs ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + } + ); + + hypothesis = addSetupTools super.hypothesis; + + importlib-metadata = addSetupTools super.importlib-metadata; + + inflect = super.inflect.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.setuptools_scm + ]; + } + ); + + jsonschema = addSetupTools super.jsonschema; + + keyring = addSetupTools super.keyring; + + lap = super.lap.overrideAttrs ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + self.numpy + ]; + } + ); + + llvmlite = super.llvmlite.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.llvm ]; + + # Disable static linking + # https://github.com/numba/llvmlite/issues/93 + postPatch = '' + substituteInPlace ffi/Makefile.linux --replace "-static-libstdc++" "" + + substituteInPlace llvmlite/tests/test_binding.py --replace "test_linux" "nope" + ''; + + # Set directory containing llvm-config binary + preConfigure = '' + export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config + ''; + + __impureHostDeps = pkgs.stdenv.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ]; + + passthru = old.passthru // { llvm = pkgs.llvm; }; + } + ); + + lockfile = super.lockfile.overrideAttrs ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pbr ]; + } + ); + + lxml = super.lxml.overrideAttrs ( + old: { + nativeBuildInputs = with pkgs; old.nativeBuildInputs ++ [ pkgconfig libxml2.dev libxslt.dev ]; + buildInputs = with pkgs; old.buildInputs ++ [ libxml2 libxslt ]; + } + ); + + markupsafe = super.markupsafe.overrideAttrs ( + old: { + src = old.src.override { pname = builtins.replaceStrings [ "markupsafe" ] [ "MarkupSafe" ] old.pname; }; + } + ); + + matplotlib = super.matplotlib.overrideAttrs ( + old: { + NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1"; + + XDG_RUNTIME_DIR = "/tmp"; + + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.pkgconfig + ]; + + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.libpng + pkgs.freetype + ]; + + inherit (super.matplotlib) patches; + } + ); + + mccabe = super.mccabe.overrideAttrs ( + old: { + postPatch = '' + substituteInPlace setup.py --replace "setup_requires=['pytest-runner']," "setup_requires=[]," || true + ''; + } + ); + + netcdf4 = super.netcdf4.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + ]; + + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.zlib + pkgs.netcdf + pkgs.hdf5 + pkgs.curl + pkgs.libjpeg + ]; + + # Variables used to configure the build process + USE_NCCONFIG = "0"; + HDF5_DIR = lib.getDev pkgs.hdf5; + NETCDF4_DIR = pkgs.netcdf; + CURL_DIR = pkgs.curl.dev; + JPEG_DIR = pkgs.libjpeg.dev; + } + ); + + numpy = super.numpy.overrideAttrs ( + old: let + blas = pkgs.openblasCompat; + blasImplementation = lib.nameFromURL blas.name "-"; + cfg = pkgs.writeTextFile { + name = "site.cfg"; + text = ( + lib.generators.toINI {} { + ${blasImplementation} = { + include_dirs = "${blas}/include"; + library_dirs = "${blas}/lib"; + } // lib.optionalAttrs (blasImplementation == "mkl") { + mkl_libs = "mkl_rt"; + lapack_libs = ""; + }; + } + ); + }; + in + { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; + buildInputs = old.buildInputs ++ [ blas ]; + enableParallelBuilding = true; + preBuild = '' + ln -s ${cfg} site.cfg + ''; + passthru = old.passthru // { + blas = blas; + inherit blasImplementation cfg; + }; + } + ); + + pillow = super.pillow.overrideAttrs ( + old: { + nativeBuildInputs = [ pkgs.pkgconfig ] ++ old.nativeBuildInputs; + buildInputs = with pkgs; [ freetype libjpeg zlib libtiff libwebp tcl lcms2 ] ++ old.buildInputs; + } + ); + + pluggy = addSetupTools super.pluggy; + + psycopg2 = super.psycopg2.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; + } + ); + + psycopg2-binary = super.psycopg2-binary.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ]; + } + ); + + py = addSetupTools super.py; + + pyarrow = super.pyarrow.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + ]; + } + ); + + pycairo = ( + drv: ( + drv.overridePythonAttrs ( + _: { + format = "other"; + } + ) + ).overrideAttrs ( + old: { + + nativeBuildInputs = old.nativeBuildInputs ++ [ + pkgs.meson + pkgs.ninja + pkgs.pkgconfig + ]; + + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + pkgs.cairo + pkgs.xlibsWrapper + ]; + + mesonFlags = [ "-Dpython=${if self.isPy3k then "python3" else "python"}" ]; + } + ) + ) super.pycairo; + + pycocotools = super.pycocotools.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.cython + self.numpy + ]; + } + ); + + pygobject = super.pygobject.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ]; + buildInputs = old.buildInputs ++ [ pkgs.glib pkgs.gobject-introspection ]; + } + ); + + pyopenssl = super.pyopenssl.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.openssl ]; + } + ); + + pytest = addSetupTools super.pytest; + + pytest-mock = addSetupTools super.pytest-mock; + + python-dateutil = addSetupTools super.python-dateutil; + + python-prctl = super.python-prctl.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ + self.setuptools_scm + pkgs.libcap + ]; + } + ); + + scaleapi = super.scaleapi.overrideAttrs ( + old: { + postPatch = '' + substituteInPlace setup.py --replace "install_requires = ['requests>=2.4.2', 'enum34']" "install_requires = ['requests>=2.4.2']" || true + ''; + } + ); + + scipy = super.scipy.overrideAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ]; + setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; + enableParallelBuilding = true; + buildInputs = old.buildInputs ++ [ self.numpy.blas ]; + preConfigure = '' + sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py + export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES + ''; + preBuild = '' + ln -s ${self.numpy.cfg} site.cfg + ''; + } + ); + + shapely = super.shapely.overrideAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.geos self.cython ]; + inherit (super.shapely) patches GEOS_LIBRARY_PATH; + } + ); + + six = addSetupTools super.six; + + urwidtrees = super.urwidtrees.overrideAttrs ( + old: { + propagatedBuildInputs = old.propagatedBuildInputs ++ [ + self.urwid + ]; + } + ); + + # TODO: Figure out getting rid of this hack + wheel = ( + pkgs.python3.pkgs.override { + python = self.python; + } + ).wheel.overridePythonAttrs ( + _: { + inherit (super.wheel) pname name version src; + } + ); + + zipp = addSetupTools super.zipp; +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix new file mode 100644 index 00000000000..b2e11205c36 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix @@ -0,0 +1,106 @@ +{ lib, stdenv, python, isLinux ? stdenv.isLinux }: + +let + inherit (lib.strings) hasSuffix hasInfix splitString removeSuffix; + + # The 'cpxy" as determined by `python.version` + # + # e.g "2.7.17" -> "cp27" + # "3.5.9" -> "cp35" + pythonTag = + let + ver = builtins.splitVersion python.version; + major = builtins.elemAt ver 0; + minor = builtins.elemAt ver 1; + in + "cp${major}${minor}"; + + abiTag = "${pythonTag}m"; + + # + # Parses wheel file returning an attribute set + # + toWheelAttrs = str: + let + entries = splitString "-" str; + p = removeSuffix ".whl" (builtins.elemAt entries 4); + in + { + pkgName = builtins.elemAt entries 0; + pkgVer = builtins.elemAt entries 1; + pyVer = builtins.elemAt entries 2; + abi = builtins.elemAt entries 3; + platform = p; + }; + + # + # Builds list of acceptable osx wheel files + # + # <versions> accepted versions in descending order of preference + # <candidates> list of wheel files to select from + findBestMatches = versions: candidates: + let + v = lib.lists.head versions; + vs = lib.lists.tail versions; + in + if (builtins.length versions == 0) + then [] + else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates); + + # pyver = "cpXX" + # x = "cpXX" | "py2" | "py3" | "py2.py3" + isPyVersionCompatible = pyver: x: + let + normalize = y: ''cp${lib.strings.removePrefix "cp" (lib.strings.removePrefix "py" y)}''; + isCompat = p: x: lib.strings.hasPrefix (normalize x) p; + in + lib.lists.any (isCompat pyver) (lib.strings.splitString "." x); + + # + # Selects the best matching wheel file from a list of files + # + selectWheel = files: + let + filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files); + + isPyAbiCompatible = pyabi: x: x == "none" || pyabi == x; + + withPython = ver: abi: x: (isPyVersionCompatible ver x.pyVer) && (isPyAbiCompatible abi x.abi); + + withPlatform = if isLinux + then ( + x: x.platform == "manylinux1_${stdenv.platform.kernelArch}" + || x.platform == "manylinux2010_${stdenv.platform.kernelArch}" + || x.platform == "manylinux2014_${stdenv.platform.kernelArch}" + || x.platform == "any" + ) + else (x: hasInfix "macosx" x.platform || x.platform == "any"); + + filterWheel = x: + let + f = toWheelAttrs x.file; + in + (withPython pythonTag abiTag f) && (withPlatform f); + + filtered = builtins.filter filterWheel filesWithoutSources; + + choose = files: + let + osxMatches = [ "10_12" "10_11" "10_10" "10_9" "any" ]; + linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ]; + chooseLinux = x: lib.singleton (builtins.head (findBestMatches linuxMatches x)); + chooseOSX = x: lib.singleton (builtins.head (findBestMatches osxMatches x)); + in + if isLinux + then chooseLinux files + else chooseOSX files; + + in + if (builtins.length filtered == 0) + then [] + else choose (filtered); + +in +{ + inherit selectWheel toWheelAttrs isPyVersionCompatible; +} diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix new file mode 100644 index 00000000000..bf1893931cd --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix @@ -0,0 +1,218 @@ +{ lib, stdenv }: python: + +let + + # Like builtins.substring but with stop being offset instead of length + substr = start: stop: s: builtins.substring start (stop - start) s; + + # Strip leading/trailing whitespace from string + stripStr = s: lib.elemAt (builtins.split "^ *" (lib.elemAt (builtins.split " *$" s) 0)) 2; + + findSubExpressionsFun = acc: c: ( + if c == "(" then ( + let + posNew = acc.pos + 1; + isOpen = acc.openP == 0; + startPos = if isOpen then posNew else acc.startPos; + in + acc // { + inherit startPos; + exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ]; + pos = posNew; + openP = acc.openP + 1; + } + ) else if c == ")" then ( + let + openP = acc.openP - 1; + exprs = findSubExpressions (substr acc.startPos acc.pos acc.expr); + in + acc // { + inherit openP; + pos = acc.pos + 1; + exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs; + exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos; + } + ) else acc // { pos = acc.pos + 1; } + ); + + # Make a tree out of expression groups (parens) + findSubExpressions = expr: let + acc = builtins.foldl' findSubExpressionsFun { + exprs = []; + expr = expr; + pos = 0; + openP = 0; + exprPos = 0; + startPos = 0; + } (lib.stringToCharacters expr); + tailExpr = (substr acc.exprPos acc.pos expr); + tailExprs = if tailExpr != "" then [ tailExpr ] else []; + in + acc.exprs ++ tailExprs; + + parseExpressions = exprs: let + splitCond = ( + s: builtins.map + (x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x)) + (builtins.split " (and|or) " (s + " ")) + ); + + mapfn = expr: ( + if (builtins.match "^ ?$" expr != null) then null # Filter empty + else if (builtins.elem expr [ "and" "or" ]) then { + type = "bool"; + value = expr; + } + else { + type = "expr"; + value = expr; + } + ); + + parse = expr: builtins.filter (x: x != null) (builtins.map mapfn (splitCond expr)); + + in + builtins.foldl' ( + acc: v: acc ++ ( + if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ] + ) + ) [] exprs; + + # Transform individual expressions to structured expressions + # This function also performs variable substitution, replacing environment markers with their explicit values + transformExpressions = exprs: let + variables = { + os_name = "posix"; # TODO: Check other platforms + sys_platform = ( + if stdenv.isLinux then "linux" + else if stdenv.isDarwin then "darwin" + else throw "Unsupported platform" + ); + platform_machine = stdenv.platform.kernelArch; + platform_python_implementation = "CPython"; # Only CPython supported for now + platform_release = ""; # Field not reproducible + platform_system = ( + if stdenv.isLinux then "Linux" + else if stdenv.isDarwin then "Darwin" + else throw "Unsupported platform" + ); + platform_version = ""; # Field not reproducible + python_version = python.passthru.pythonVersion; + python_full_version = python.version; + implementation_name = "cpython"; # Only cpython supported for now + implementation_version = python.version; + extra = ""; + }; + + substituteVar = value: if builtins.hasAttr value variables then (builtins.toJSON variables."${value}") else value; + + processVar = value: builtins.foldl' (acc: v: v acc) value [ + stripStr + substituteVar + ]; + + in + if builtins.typeOf exprs == "set" then ( + if exprs.type == "expr" then ( + let + mVal = ''[a-zA-Z0-9\'"_\. ]+''; + mOp = "in|[!=<>]+"; + e = stripStr exprs.value; + m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e); + in + { + type = "expr"; + value = { + op = builtins.elemAt m 1; + values = [ + (processVar (builtins.elemAt m 0)) + (processVar (builtins.elemAt m 2)) + ]; + }; + } + ) else exprs + ) else builtins.map transformExpressions exprs; + + # Recursively eval all expressions + evalExpressions = exprs: let + unmarshal = v: ( + # TODO: Handle single quoted values + if v == "True" then true + else if v == "False" then false + else builtins.fromJSON v + ); + hasElem = needle: haystack: builtins.elem needle (builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " haystack)); + # TODO: Implement all operators + op = { + "<=" = x: y: (unmarshal x) <= (unmarshal y); + "<" = x: y: (unmarshal x) < (unmarshal y); + "!=" = x: y: x != y; + "==" = x: y: x == y; + ">=" = x: y: (unmarshal x) >= (unmarshal y); + ">" = x: y: (unmarshal x) > (unmarshal y); + "~=" = null; + "===" = null; + "in" = x: y: let + values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y)); + in + builtins.elem (unmarshal x) values; + }; + in + if builtins.typeOf exprs == "set" then ( + if exprs.type == "expr" then ( + let + expr = exprs; + result = (op."${expr.value.op}") (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1); + in + { + type = "value"; + value = result; + } + ) else exprs + ) else builtins.map evalExpressions exprs; + + # Now that we have performed an eval all that's left to do is to concat the graph into a single bool + reduceExpressions = exprs: let + cond = { + "and" = x: y: x && y; + "or" = x: y: x || y; + }; + reduceExpressionsFun = acc: v: ( + if builtins.typeOf v == "set" then ( + if v.type == "value" then ( + acc // { + value = cond."${acc.cond}" acc.value v.value; + } + ) else if v.type == "bool" then ( + acc // { + cond = v.value; + } + ) else throw "Unsupported type" + ) else if builtins.typeOf v == "list" then ( + let + ret = builtins.foldl' reduceExpressionsFun { + value = true; + cond = "and"; + } v; + in + acc // { + value = cond."${acc.cond}" acc.value ret.value; + } + ) else throw "Unsupported type" + ); + in + ( + builtins.foldl' reduceExpressionsFun { + value = true; + cond = "and"; + } exprs + ).value; + +in +e: builtins.foldl' (acc: v: v acc) e [ + findSubExpressions + parseExpressions + transformExpressions + evalExpressions + reduceExpressions +] diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py b/pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py new file mode 100644 index 00000000000..bb61e4a5cb4 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# Patch out path dependencies from a pyproject.json file + +import json +import sys + +data = json.load(sys.stdin) + +for dep in data['tool']['poetry']['dependencies'].values(): + if isinstance(dep, dict): + try: + del dep['path']; + except KeyError: + pass + else: + dep['version'] = '*' + +json.dump(data, sys.stdout, indent=4) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix new file mode 100644 index 00000000000..620bb25ad97 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/semver.nix @@ -0,0 +1,80 @@ +{ lib }: + +let + inherit (builtins) elemAt match; + + # Replace a list entry at defined index with set value + ireplace = idx: value: list: let + inherit (builtins) genList length; + in + genList (i: if i == idx then value else (elemAt list i)) (length list); + + operators = let + matchWildCard = s: match "([^\*])(\.[\*])" s; + mkComparison = ret: version: v: builtins.compareVersions version v == ret; + mkIdxComparison = idx: version: v: let + ver = builtins.splitVersion v; + minor = builtins.toString (lib.toInt (elemAt ver idx) + 1); + upper = builtins.concatStringsSep "." (ireplace idx minor ver); + in + operators.">=" version v && operators."<" version upper; + dropWildcardPrecision = f: version: constraint: let + m = matchWildCard constraint; + hasWildcard = m != null; + c = if hasWildcard then (elemAt m 0) else constraint; + v = + if hasWildcard then (builtins.substring 0 (builtins.stringLength c) version) + else version; + in + f v c; + in + { + # Prefix operators + "==" = dropWildcardPrecision (mkComparison 0); + ">" = dropWildcardPrecision (mkComparison 1); + "<" = dropWildcardPrecision (mkComparison (-1)); + "!=" = v: c: ! operators."==" v c; + ">=" = v: c: operators."==" v c || operators.">" v c; + "<=" = v: c: operators."==" v c || operators."<" v c; + # Semver specific operators + "~" = mkIdxComparison 1; # + "^" = mkIdxComparison 0; + # Infix operators + "-" = version: v: operators.">=" version v.vl && operators."<=" version v.vu; + }; + + re = { + operators = "([=><!~\^]+)"; + version = "([0-9\.\*x]+)"; + }; + + parseConstraint = constraint: let + constraintStr = builtins.replaceStrings [ " " ] [ "" ] constraint; + # The common prefix operators + mPre = match "${re.operators} *${re.version}" constraintStr; + # There is also an infix operator to match ranges + mIn = match "${re.version} *(-) *${re.version}" constraintStr; + in + ( + if mPre != null then { + op = elemAt mPre 0; + v = elemAt mPre 1; + } + # Infix operators are range matches + else if mIn != null then { + op = elemAt mIn 1; + v = { + vl = (elemAt mIn 0); + vu = (elemAt mIn 2); + }; + } + else throw "Constraint \"${constraintStr}\" could not be parsed" + ); + + satisfiesSemver = version: constraint: let + inherit (parseConstraint constraint) op v; + in + if constraint == "*" then true else operators."${op}" version v; + +in +{ inherit satisfiesSemver; } diff --git a/pkgs/development/tools/poetry2nix/update b/pkgs/development/tools/poetry2nix/update new file mode 100755 index 00000000000..a5d367590be --- /dev/null +++ b/pkgs/development/tools/poetry2nix/update @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +pwd=$(pwd) +workdir=$(mktemp -d) + +function cleanup { + cd "$pwd" + rm -rf $workdir +} +trap cleanup EXIT + +cd "$workdir" + +curl -L -s https://github.com/nix-community/poetry2nix/archive/master.tar.gz | tar -xz +mv poetry2nix-master/* . + +mkdir build +cp *.nix *.json *.py build/ +cp -r bin build/ +rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix + +cat > build/README.md << EOF +Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix + +The update procedure is as-follows: +1. Send your change to the upstream poetry2nix repository +2. Get it approved with tests passing +3. Run the update script in pkgs/development/tools/poetry2nix +EOF + +rm -rf "$pwd/poetry2nix" +mv build "$pwd/poetry2nix" |