diff options
Diffstat (limited to 'pkgs/misc/vim-plugins/update.py')
| -rwxr-xr-x | pkgs/misc/vim-plugins/update.py | 138 |
1 files changed, 109 insertions, 29 deletions
diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index 8a8e20da8d7..0ef93ac569a 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -8,6 +8,7 @@ # linted: # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py +import argparse import functools import json import os @@ -20,15 +21,53 @@ import xml.etree.ElementTree as ET from datetime import datetime from multiprocessing.dummy import Pool from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union, Any +from typing import Dict, List, Optional, Tuple, Union, Any, Callable from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile -ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" -ATOM_LINK = "{http://www.w3.org/2005/Atom}link" -ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" +ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here +ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " +ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " ROOT = Path(__file__).parent +DEFAULT_IN = ROOT.joinpath("vim-plugin-names") +DEFAULT_OUT = ROOT.joinpath("generated.nix") + +import time +from functools import wraps + + +def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): + """Retry calling the decorated function using an exponential backoff. + + http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ + original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry + (BSD licensed) + + :param ExceptionToCheck: the exception on which to retry + :param tries: number of times to try (not retry) before giving up + :param delay: initial delay between retries in seconds + :param backoff: backoff multiplier e.g. value of 2 will double the delay + each retry + """ + + def deco_retry(f: Callable) -> Callable: + @wraps(f) + def f_retry(*args: Any, **kwargs: Any) -> Any: + mtries, mdelay = tries, delay + while mtries > 1: + try: + return f(*args, **kwargs) + except ExceptionToCheck as e: + print(f"{str(e)}, Retrying in {mdelay} seconds...") + time.sleep(mdelay) + mtries -= 1 + mdelay *= backoff + return f(*args, **kwargs) + + return f_retry # true decorator + + return deco_retry class Repo: @@ -42,9 +81,12 @@ class Repo: def __repr__(self) -> str: return f"Repo({self.owner}, {self.name})" + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def has_submodules(self) -> bool: try: - urllib.request.urlopen(self.url("blob/master/.gitmodules")).close() + urllib.request.urlopen( + self.url("blob/master/.gitmodules"), timeout=10 + ).close() except urllib.error.HTTPError as e: if e.code == 404: return False @@ -52,8 +94,9 @@ class Repo: raise return True + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: - with urllib.request.urlopen(self.url("commits/master.atom")) as req: + with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req: xml = req.read() root = ET.fromstring(xml) latest_entry = root.find(ATOM_ENTRY) @@ -66,7 +109,7 @@ class Repo: updated_tag is not None and updated_tag.text is not None ), f"No updated tag found feed entry {xml}" updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ") - return Path(url.path).name, updated + return Path(str(url.path)).name, updated def prefetch_git(self, ref: str) -> str: data = subprocess.check_output( @@ -154,13 +197,13 @@ def get_current_plugins() -> List[Plugin]: return plugins -def prefetch_plugin(user: str, repo_name: str, cache: "Cache") -> Plugin: +def prefetch_plugin(user: str, repo_name: str, alias: str, cache: "Cache") -> Plugin: repo = Repo(user, repo_name) commit, date = repo.latest_commit() has_submodules = repo.has_submodules() cached_plugin = cache[commit] if cached_plugin is not None: - cached_plugin.name = repo_name + cached_plugin.name = alias or repo_name cached_plugin.date = date return cached_plugin @@ -170,7 +213,7 @@ def prefetch_plugin(user: str, repo_name: str, cache: "Cache") -> Plugin: else: sha256 = repo.prefetch_github(commit) - return Plugin(repo_name, commit, has_submodules, sha256, date=date) + return Plugin(alias or repo_name, commit, has_submodules, sha256, date=date) def print_download_error(plugin: str, ex: Exception): @@ -207,18 +250,26 @@ def check_results( sys.exit(1) -def load_plugin_spec() -> List[Tuple[str, str]]: - plugin_file = ROOT.joinpath("vim-plugin-names") +def parse_plugin_line(line: str) -> Tuple[str, str, Optional[str]]: + name, repo = line.split("/") + try: + repo, alias = repo.split(" as ") + return (name, repo, alias.strip()) + except ValueError: + # no alias defined + return (name, repo.strip(), None) + + +def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, Optional[str]]]: plugins = [] with open(plugin_file) as f: for line in f: - spec = line.strip() - parts = spec.split("/") - if len(parts) != 2: - msg = f"Invalid repository {spec}, must be in the format owner/repo" + plugin = parse_plugin_line(line) + if not plugin[0]: + msg = f"Invalid repository {line}, must be in the format owner/repo[ as alias]" print(msg, file=sys.stderr) sys.exit(1) - plugins.append((parts[0], parts[1])) + plugins.append(plugin) return plugins @@ -276,12 +327,12 @@ class Cache: def prefetch( - args: Tuple[str, str], cache: Cache + args: Tuple[str, str, str], cache: Cache ) -> Tuple[str, str, Union[Exception, Plugin]]: - assert len(args) == 2 - owner, repo = args + assert len(args) == 3 + owner, repo, alias = args try: - plugin = prefetch_plugin(owner, repo, cache) + plugin = prefetch_plugin(owner, repo, alias, cache) cache[plugin.commit] = plugin return (owner, repo, plugin) except Exception as e: @@ -293,10 +344,10 @@ header = ( ) -def generate_nix(plugins: List[Tuple[str, str, Plugin]]): +def generate_nix(plugins: List[Tuple[str, str, Plugin]], outfile: str): sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower()) - with open(ROOT.joinpath("generated.nix"), "w+") as f: + with open(outfile, "w+") as f: f.write(header) f.write( """ @@ -326,15 +377,44 @@ let }}; """ ) - f.write(""" + f.write( + """ }); in lib.fix' (lib.extends overrides packages) -""") - print("updated generated.nix") +""" + ) + print(f"updated {outfile}") + + +def parse_args(): + parser = argparse.ArgumentParser( + description=( + "Updates nix derivations for vim plugins" + f"By default from {DEFAULT_IN} to {DEFAULT_OUT}" + ) + ) + parser.add_argument( + "--input-names", + "-i", + dest="input_file", + default=DEFAULT_IN, + help="A list of plugins in the form owner/repo", + ) + parser.add_argument( + "--out", + "-o", + dest="outfile", + default=DEFAULT_OUT, + help="Filename to save generated nix code", + ) + + return parser.parse_args() def main() -> None: - plugin_names = load_plugin_spec() + + args = parse_args() + plugin_names = load_plugin_spec(args.input_file) current_plugins = get_current_plugins() cache = Cache(current_plugins) @@ -342,7 +422,7 @@ def main() -> None: try: # synchronous variant for debugging - # results = map(prefetch_with_cache, plugins) + # results = list(map(prefetch_with_cache, plugin_names)) pool = Pool(processes=30) results = pool.map(prefetch_with_cache, plugin_names) finally: @@ -350,7 +430,7 @@ def main() -> None: plugins = check_results(results) - generate_nix(plugins) + generate_nix(plugins, args.outfile) if __name__ == "__main__": |