diff options
author | Martin Weinelt <hexa@darmstadt.ccc.de> | 2023-02-12 17:20:42 +0100 |
---|---|---|
committer | Martin Weinelt <mweinelt@users.noreply.github.com> | 2023-02-27 02:21:07 +0000 |
commit | 706cae653224fcec15a2d27c539bd0437e7e9dcf (patch) | |
tree | d458b1ea683fecc650071c64f9feb86c641bb51e /pkgs/development/interpreters/python | |
parent | 5d447a9e7009a116ab1e62dd5599b9272206067f (diff) | |
download | nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar.gz nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar.bz2 nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar.lz nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar.xz nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.tar.zst nixpkgs-706cae653224fcec15a2d27c539bd0437e7e9dcf.zip |
update-python-libraries: Add support for fetchgit
Previously when packages that required the git fetcher were updated, we would wrongly rely on `nix-prefetch-url`, which would reliable break the hash. Instead we need to use `nix-prefetch-git` to determine the proper hash, when the relevant attributes are present.
Diffstat (limited to 'pkgs/development/interpreters/python')
-rw-r--r-- | pkgs/development/interpreters/python/update-python-libraries/default.nix | 3 | ||||
-rwxr-xr-x | pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py | 78 |
2 files changed, 65 insertions, 16 deletions
diff --git a/pkgs/development/interpreters/python/update-python-libraries/default.nix b/pkgs/development/interpreters/python/update-python-libraries/default.nix index 497799e0a2a..f7dfdf0a8b4 100644 --- a/pkgs/development/interpreters/python/update-python-libraries/default.nix +++ b/pkgs/development/interpreters/python/update-python-libraries/default.nix @@ -1,8 +1,9 @@ -{ python3, runCommand, git, nix }: +{ python3, runCommand, git, nix, nix-prefetch-git }: runCommand "update-python-libraries" { buildInputs = [ nix + nix-prefetch-git (python3.withPackages(ps: with ps; [ packaging requests toolz ])) git ]; diff --git a/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py b/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py index feb44bef079..396a5538582 100755 --- a/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py +++ b/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py @@ -12,6 +12,7 @@ to update all non-pinned libraries in that folder. """ import argparse +import json import os import pathlib import re @@ -20,6 +21,7 @@ from concurrent.futures import ThreadPoolExecutor as Pool from packaging.version import Version as _Version from packaging.version import InvalidVersion from packaging.specifiers import SpecifierSet +from typing import Optional, Any import collections import subprocess @@ -33,7 +35,7 @@ PRERELEASES = False GIT = "git" -NIXPGKS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip() +NIXPKGS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip() import logging logging.basicConfig(level=logging.INFO) @@ -67,6 +69,22 @@ def _get_values(attribute, text): values = regex.findall(text) return values + +def _get_attr_value(attr_path: str) -> Optional[Any]: + try: + response = subprocess.check_output([ + "nix", + "--extra-experimental-features", "nix-command", + "eval", + "-f", f"{NIXPKGS_ROOT}/default.nix", + "--json", + f"{attr_path}" + ]) + return json.loads(response.decode()) + except (subprocess.CalledProcessError, ValueError): + return None + + def _get_unique_value(attribute, text): """Match attribute in text and return unique match. @@ -124,6 +142,16 @@ def _fetch_github(url): raise ValueError("request for {} failed".format(url)) +def _hash_to_sri(algorithm, value): + """Convert a hash to its SRI representation""" + return subprocess.check_output([ + "nix", + "hash", + "to-sri", + "--type", algorithm, + value + ]).decode().strip() + SEMVER = { 'major' : 0, 'minor' : 1, @@ -198,7 +226,7 @@ def _get_latest_version_github(package, extension, current_version, target): attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{package}") try: homepage = subprocess.check_output( - ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\ + ["nix", "eval", "-f", f"{NIXPKGS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\ .decode('utf-8') except Exception as e: raise ValueError(f"Unable to determine homepage: {e}") @@ -217,17 +245,39 @@ def _get_latest_version_github(package, extension, current_version, target): release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases)) prefix = get_prefix(release['tag_name']) - try: - sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\ - .decode('utf-8').strip() - except: - # this may fail if they have both a branch and a tag of the same name, attempt tag name - tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags") - sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], stderr=subprocess.DEVNULL)\ - .decode('utf-8').strip() + # some attributes require using the fetchgit + git_fetcher_args = [] + if (_get_attr_value(f"{attr_path}.src.fetchSubmodules")): + git_fetcher_args.append("--fetch-submodules") + if (_get_attr_value(f"{attr_path}.src.fetchLFS")): + git_fetcher_args.append("--fetch-lfs") + if (_get_attr_value(f"{attr_path}.src.leaveDotGit")): + git_fetcher_args.append("--leave-dotGit") + + if git_fetcher_args: + algorithm = "sha256" + cmd = [ + "nix-prefetch-git", + f"https://github.com/{owner}/{repo}.git", + "--hash", algorithm, + "--rev", f"refs/tags/{release['tag_name']}" + ] + cmd.extend(git_fetcher_args) + response = subprocess.check_output(cmd) + document = json.loads(response.decode()) + hash = _hash_to_sri(algorithm, document[algorithm]) + else: + try: + hash = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\ + .decode('utf-8').strip() + except: + # this may fail if they have both a branch and a tag of the same name, attempt tag name + tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags") + hash = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], stderr=subprocess.DEVNULL)\ + .decode('utf-8').strip() - return version, sha256, prefix + return version, hash, prefix FETCHERS = { @@ -294,8 +344,6 @@ def _determine_extension(text, fetcher): raise ValueError('url does not point to PyPI.') elif fetcher == 'fetchFromGitHub': - if "fetchSubmodules" in text: - raise ValueError("fetchFromGitHub fetcher doesn't support submodules") extension = "tar.gz" return extension @@ -340,10 +388,10 @@ def _update_package(path, target): raise ValueError("no file available for {}.".format(pname)) text = _replace_value('version', new_version, text) + # hashes from pypi are 16-bit encoded sha256's, normalize it to sri to avoid merge conflicts # sri hashes have been the default format since nix 2.4+ - sri_hash = subprocess.check_output(["nix", "--extra-experimental-features", "nix-command", "hash", "to-sri", "--type", "sha256", new_sha256]).decode('utf-8').strip() - + sri_hash = _hash_to_sri("sha256", new_sha256) # fetchers can specify a sha256, or a sri hash try: |