From 2c9ecb4f4d3df678d78154cdd0c97ab86fb03718 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 26 Nov 2017 09:08:49 +0100 Subject: update-python-libraries: commit updates and specify update kind This commit introduces two new features: 1. specify with --target whether major, minor or patch updates should be made 2. use --commit to create commits for each of the updates --- maintainers/scripts/update-python-libraries | 160 ++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 22 deletions(-) (limited to 'maintainers/scripts/update-python-libraries') diff --git a/maintainers/scripts/update-python-libraries b/maintainers/scripts/update-python-libraries index 3ddc8c23a79..ec2691ff617 100755 --- a/maintainers/scripts/update-python-libraries +++ b/maintainers/scripts/update-python-libraries @@ -1,5 +1,5 @@ #! /usr/bin/env nix-shell -#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])' +#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ packaging requests toolz ])' -p git """ Update a Python package expression by passing in the `.nix` file, or the directory containing it. @@ -18,7 +18,12 @@ import os import re import requests import toolz -from concurrent.futures import ThreadPoolExecutor as pool +from concurrent.futures import ThreadPoolExecutor as Pool +from packaging.version import Version as _Version +from packaging.version import InvalidVersion +from packaging.specifiers import SpecifierSet +import collections +import subprocess INDEX = "https://pypi.io/pypi" """url of PyPI""" @@ -26,10 +31,30 @@ INDEX = "https://pypi.io/pypi" EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl'] """Permitted file extensions. These are evaluated from left to right and the first occurance is returned.""" +PRERELEASES = False + import logging logging.basicConfig(level=logging.INFO) +class Version(_Version, collections.abc.Sequence): + + def __init__(self, version): + super().__init__(version) + # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21` + # https://github.com/avian2/unidecode/issues/13#issuecomment-354538882 + self.raw_version = version + + def __getitem__(self, i): + return self._version.release[i] + + def __len__(self): + return len(self._version.release) + + def __iter__(self): + yield from self._version.release + + def _get_values(attribute, text): """Match attribute in text and return all matches. @@ -82,13 +107,59 @@ def _fetch_page(url): else: raise ValueError("request for {} failed".format(url)) -def _get_latest_version_pypi(package, extension): + +SEMVER = { + 'major' : 0, + 'minor' : 1, + 'patch' : 2, +} + + +def _determine_latest_version(current_version, target, versions): + """Determine latest version, given `target`. + """ + current_version = Version(current_version) + + def _parse_versions(versions): + for v in versions: + try: + yield Version(v) + except InvalidVersion: + pass + + versions = _parse_versions(versions) + + index = SEMVER[target] + + ceiling = list(current_version[0:index]) + if len(ceiling) == 0: + ceiling = None + else: + ceiling[-1]+=1 + ceiling = Version(".".join(map(str, ceiling))) + + # We do not want prereleases + versions = SpecifierSet(prereleases=PRERELEASES).filter(versions) + + if ceiling is not None: + versions = SpecifierSet(f"<{ceiling}").filter(versions) + + return (max(sorted(versions))).raw_version + + +def _get_latest_version_pypi(package, extension, current_version, target): """Get latest version and hash from PyPI.""" url = "{}/{}/json".format(INDEX, package) json = _fetch_page(url) - version = json['info']['version'] - for release in json['releases'][version]: + versions = json['releases'].keys() + version = _determine_latest_version(current_version, target, versions) + + try: + releases = json['releases'][version] + except KeyError as e: + raise KeyError('Could not find version {} for {}'.format(version, package)) from e + for release in releases: if release['filename'].endswith(extension): # TODO: In case of wheel we need to do further checks! sha256 = release['digests']['sha256'] @@ -98,7 +169,7 @@ def _get_latest_version_pypi(package, extension): return version, sha256 -def _get_latest_version_github(package, extension): +def _get_latest_version_github(package, extension, current_version, target): raise ValueError("updating from GitHub is not yet supported.") @@ -141,9 +212,9 @@ def _determine_extension(text, fetcher): """ if fetcher == 'fetchPypi': try: - format = _get_unique_value('format', text) + src_format = _get_unique_value('format', text) except ValueError as e: - format = None # format was not given + src_format = None # format was not given try: extension = _get_unique_value('extension', text) @@ -151,9 +222,11 @@ def _determine_extension(text, fetcher): extension = None # extension was not given if extension is None: - if format is None: - format = 'setuptools' - extension = FORMATS[format] + if src_format is None: + src_format = 'setuptools' + elif src_format == 'flit': + raise ValueError("Don't know how to update a Flit package.") + extension = FORMATS[src_format] elif fetcher == 'fetchurl': url = _get_unique_value('url', text) @@ -167,9 +240,7 @@ def _determine_extension(text, fetcher): return extension -def _update_package(path): - - +def _update_package(path, target): # Read the expression with open(path, 'r') as f: @@ -186,11 +257,13 @@ def _update_package(path): extension = _determine_extension(text, fetcher) - new_version, new_sha256 = _get_latest_version_pypi(pname, extension) + new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target) if new_version == version: logging.info("Path {}: no update available for {}.".format(path, pname)) return False + elif new_version <= version: + raise ValueError("downgrade for {}.".format(pname)) if not new_sha256: raise ValueError("no file available for {}.".format(pname)) @@ -202,10 +275,19 @@ def _update_package(path): logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version)) - return True + result = { + 'path' : path, + 'target': target, + 'pname': pname, + 'old_version' : version, + 'new_version' : new_version, + #'fetcher' : fetcher, + } + return result -def _update(path): + +def _update(path, target): # We need to read and modify a Nix expression. if os.path.isdir(path): @@ -222,24 +304,58 @@ def _update(path): return False try: - return _update_package(path) + return _update_package(path, target) except ValueError as e: logging.warning("Path {}: {}".format(path, e)) return False + +def _commit(path, pname, old_version, new_version, **kwargs): + """Commit result. + """ + + msg = f'python: {pname}: {old_version} -> {new_version}' + + try: + subprocess.check_call(['git', 'add', path]) + subprocess.check_call(['git', 'commit', '-m', msg]) + except subprocess.CalledProcessError as e: + subprocess.check_call(['git', 'checkout', path]) + raise subprocess.CalledProcessError(f'Could not commit {path}') from e + + return True + + def main(): parser = argparse.ArgumentParser() parser.add_argument('package', type=str, nargs='+') + parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major') + parser.add_argument('--commit', action='store_true', help='Create a commit for each package update') args = parser.parse_args() + target = args.target + + packages = list(map(os.path.abspath, args.package)) + + logging.info("Updating packages...") + + # Use threads to update packages concurrently + with Pool() as p: + results = list(p.map(lambda pkg: _update(pkg, target), packages)) + + logging.info("Finished updating packages.") + + # Commits are created sequentially. + if args.commit: + logging.info("Committing updates...") + list(map(lambda x: _commit(**x), filter(bool, results))) + logging.info("Finished committing updates") - packages = map(os.path.abspath, args.package) + count = sum(map(bool, results)) + logging.info("{} package(s) updated".format(count)) - with pool() as p: - count = list(p.map(_update, packages)) - logging.info("{} package(s) updated".format(sum(count))) if __name__ == '__main__': main() \ No newline at end of file -- cgit 1.4.1