From ec6cf1aa5198e04b75c0ec72bea1bffc3df06994 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Wed, 30 Mar 2022 01:54:26 +0200 Subject: update.py: updater vim updater We want to support plugins on other hosts than github. A more structured format will allow more features too. --- maintainers/scripts/pluginupdate.py | 187 ++++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 83 deletions(-) (limited to 'maintainers') diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py index 017e3ac758a..3cfdb138705 100644 --- a/maintainers/scripts/pluginupdate.py +++ b/maintainers/scripts/pluginupdate.py @@ -8,6 +8,7 @@ # $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py import argparse +import csv import functools import http import json @@ -28,7 +29,7 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union, Any, Callable from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile -from dataclasses import dataclass +from dataclasses import dataclass, asdict import git @@ -85,21 +86,30 @@ def make_request(url: str, token=None) -> urllib.request.Request: headers["Authorization"] = f"token {token}" return urllib.request.Request(url, headers=headers) + +Redirects = Dict['Repo', 'Repo'] + class Repo: def __init__( - self, uri: str, branch: str, alias: Optional[str] + self, uri: str, branch: str ) -> None: self.uri = uri '''Url to the repo''' - self.branch = branch - self.alias = alias - self.redirect: Dict[str, str] = {} + self._branch = branch + # {old_uri: new_uri} + self.redirect: Redirects = {} self.token = "dummy_token" @property def name(self): return self.uri.split('/')[-1] + @property + def branch(self): + return self._branch or "HEAD" + + def __str__(self) -> str: + return f"{self.uri}" def __repr__(self) -> str: return f"Repo({self.name}, {self.uri})" @@ -109,6 +119,7 @@ class Repo: @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: + log.debug("Latest commit") loaded = self._prefetch(None) updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z") @@ -124,6 +135,7 @@ class Repo: return loaded def prefetch(self, ref: Optional[str]) -> str: + print("Prefetching") loaded = self._prefetch(ref) return loaded["sha256"] @@ -137,21 +149,22 @@ class Repo: class RepoGitHub(Repo): def __init__( - self, owner: str, repo: str, branch: str, alias: Optional[str] + self, owner: str, repo: str, branch: str ) -> None: self.owner = owner self.repo = repo self.token = None '''Url to the repo''' - super().__init__(self.url(""), branch, alias) - log.debug("Instantiating github repo %s/%s", self.owner, self.repo) + super().__init__(self.url(""), branch) + log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo) @property def name(self): return self.repo def url(self, path: str) -> str: - return urljoin(f"https://github.com/{self.owner}/{self.name}/", path) + res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path) + return res @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def has_submodules(self) -> bool: @@ -168,6 +181,7 @@ class RepoGitHub(Repo): @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: commit_url = self.url(f"commits/{self.branch}.atom") + log.debug("Sending request to %s", commit_url) commit_req = make_request(commit_url, self.token) with urllib.request.urlopen(commit_req, timeout=10) as req: self._check_for_redirect(commit_url, req) @@ -191,12 +205,9 @@ class RepoGitHub(Repo): new_owner, new_name = ( urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2] ) - end_line = "\n" if self.alias is None else f" as {self.alias}\n" - plugin_line = "{owner}/{name}" + end_line - old_plugin = plugin_line.format(owner=self.owner, name=self.name) - new_plugin = plugin_line.format(owner=new_owner, name=new_name) - self.redirect[old_plugin] = new_plugin + new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch) + self.redirect[self] = new_repo def prefetch(self, commit: str) -> str: @@ -207,9 +218,9 @@ class RepoGitHub(Repo): return sha256 def prefetch_github(self, ref: str) -> str: - data = subprocess.check_output( - ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")] - ) + cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")] + log.debug("Running %s", cmd) + data = subprocess.check_output(cmd) return data.strip().decode("utf-8") def as_nix(self, plugin: "Plugin") -> str: @@ -239,21 +250,38 @@ class PluginDesc: else: return self.alias + def __lt__(self, other): + return self.repo.name < other.repo.name + + @staticmethod + def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc': + branch = row["branch"] + repo = make_repo(row['repo'], branch.strip()) + repo.token = config.github_token + return PluginDesc(repo, branch.strip(), row["alias"]) + + + @staticmethod + def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc': + branch = "HEAD" + alias = None + uri = line + if " as " in uri: + uri, alias = uri.split(" as ") + alias = alias.strip() + if "@" in uri: + uri, branch = uri.split("@") + repo = make_repo(uri.strip(), branch.strip()) + repo.token = config.github_token + return PluginDesc(repo, branch.strip(), alias) +@dataclass class Plugin: - def __init__( - self, - name: str, - commit: str, - has_submodules: bool, - sha256: str, - date: Optional[datetime] = None, - ) -> None: - self.name = name - self.commit = commit - self.has_submodules = has_submodules - self.sha256 = sha256 - self.date = date + name: str + commit: str + has_submodules: bool + sha256: str + date: Optional[datetime] = None @property def normalized_name(self) -> str: @@ -270,6 +298,17 @@ class Plugin: return copy +def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]: + log.debug("Load plugins from csv %s", input_file) + plugins = [] + with open(input_file, newline='') as csvfile: + log.debug("Writing into %s", input_file) + reader = csv.DictReader(csvfile,) + for line in reader: + plugin = PluginDesc.load_from_csv(config, line) + plugins.append(plugin) + + return plugins class Editor: """The configuration of the update script.""" @@ -298,14 +337,8 @@ class Editor: return get_current_plugins(self) def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]: - plugins = [] - with open(plugin_file) as f: - for line in f: - if line.startswith("#"): - continue - plugin = parse_plugin_line(config, line) - plugins.append(plugin) - return plugins + '''CSV spec''' + return load_plugins_from_csv(config, plugin_file) def generate_nix(self, plugins, outfile: str): '''Returns nothing for now, writes directly to outfile''' @@ -316,11 +349,11 @@ class Editor: _prefetch = functools.partial(prefetch, cache=cache) def update() -> dict: - plugin_names = self.load_plugin_spec(config, input_file) + plugins = self.load_plugin_spec(config, input_file) try: pool = Pool(processes=config.proc) - results = pool.map(_prefetch, plugin_names) + results = pool.map(_prefetch, plugins) finally: cache.store() @@ -423,6 +456,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]: data = json.loads(out) plugins = [] for name, attr in data.items(): + print("get_current_plugins: name %s" % name) p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"]) plugins.append(p) return plugins @@ -431,7 +465,7 @@ def get_current_plugins(editor: Editor) -> List[Plugin]: def prefetch_plugin( p: PluginDesc, cache: "Optional[Cache]" = None, -) -> Tuple[Plugin, Dict[str, str]]: +) -> Tuple[Plugin, Redirects]: repo, branch, alias = p.repo, p.branch, p.alias name = alias or p.repo.name commit = None @@ -454,11 +488,6 @@ def prefetch_plugin( ) -def fetch_plugin_from_pluginline(config: FetchConfig, plugin_line: str) -> Plugin: - plugin, _ = prefetch_plugin(parse_plugin_line(config, plugin_line)) - return plugin - - def print_download_error(plugin: str, ex: Exception): print(f"{plugin}: {ex}", file=sys.stderr) ex_traceback = ex.__traceback__ @@ -468,14 +497,14 @@ def print_download_error(plugin: str, ex: Exception): ] print("\n".join(tb_lines)) - def check_results( - results: List[Tuple[PluginDesc, Union[Exception, Plugin], Dict[str, str]]] -) -> Tuple[List[Tuple[PluginDesc, Plugin]], Dict[str, str]]: + results: List[Tuple[PluginDesc, Union[Exception, Plugin], Redirects]] +) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]: ''' ''' failures: List[Tuple[str, Exception]] = [] plugins = [] - redirects: Dict[str, str] = {} + # {old: new} plugindesc + redirects: Dict[Repo, Repo] = {} for (pdesc, result, redirect) in results: if isinstance(result, Exception): failures.append((pdesc.name, result)) @@ -495,31 +524,17 @@ def check_results( sys.exit(1) -def make_repo(uri, branch, alias) -> Repo: +def make_repo(uri: str, branch) -> Repo: '''Instantiate a Repo with the correct specialization depending on server (gitub spec)''' # dumb check to see if it's of the form owner/repo (=> github) or https://... - res = uri.split('/') - if len(res) <= 2: - repo = RepoGitHub(res[0], res[1], branch, alias) + res = urlparse(uri) + if res.netloc in [ "github.com", ""]: + res = res.path.strip('/').split('/') + repo = RepoGitHub(res[0], res[1], branch) else: - repo = Repo(uri.strip(), branch, alias) + repo = Repo(uri.strip(), branch) return repo -def parse_plugin_line(config: FetchConfig, line: str) -> PluginDesc: - branch = "HEAD" - alias = None - uri = line - if " as " in uri: - uri, alias = uri.split(" as ") - alias = alias.strip() - if "@" in uri: - uri, branch = uri.split("@") - - repo = make_repo(uri.strip(), branch.strip(), alias) - repo.token = config.github_token - - return PluginDesc(repo, branch.strip(), alias) - def get_cache_path(cache_file_name: str) -> Optional[Path]: xdg_cache = os.environ.get("XDG_CACHE_HOME", None) @@ -585,27 +600,27 @@ def prefetch( return (pluginDesc, e, {}) + def rewrite_input( config: FetchConfig, input_file: Path, deprecated: Path, - redirects: Dict[str, str] = None, - append: Tuple = (), + # old pluginDesc and the new + redirects: Dict[PluginDesc, PluginDesc] = {}, + append: List[PluginDesc] = [], ): - with open(input_file, "r") as f: - lines = f.readlines() + plugins = load_plugins_from_csv(config, input_file,) - lines.extend(append) + plugins.extend(append) if redirects: - lines = [redirects.get(line, line) for line in lines] cur_date_iso = datetime.now().strftime("%Y-%m-%d") with open(deprecated, "r") as f: deprecations = json.load(f) for old, new in redirects.items(): - old_plugin = fetch_plugin_from_pluginline(config, old) - new_plugin = fetch_plugin_from_pluginline(config, new) + old_plugin, _ = prefetch_plugin(old) + new_plugin, _ = prefetch_plugin(new) if old_plugin.normalized_name != new_plugin.normalized_name: deprecations[old_plugin.normalized_name] = { "new": new_plugin.normalized_name, @@ -615,10 +630,14 @@ def rewrite_input( json.dump(deprecations, f, indent=4, sort_keys=True) f.write("\n") - lines = sorted(lines, key=str.casefold) - with open(input_file, "w") as f: - f.writelines(lines) + log.debug("Writing into %s", input_file) + # fields = dataclasses.fields(PluginDesc) + fieldnames = ['repo', 'branch', 'alias'] + writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE) + writer.writeheader() + for plugin in sorted(plugins): + writer.writerow(asdict(plugin)) def commit(repo: git.Repo, message: str, files: List[Path]) -> None: @@ -660,9 +679,11 @@ def update_plugins(editor: Editor, args): ) for plugin_line in args.add_plugins: - editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=(plugin_line + "\n",)) + pdesc = PluginDesc.load_from_string(fetch_config, plugin_line) + append = [ pdesc ] + editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append) update() - plugin = fetch_plugin_from_pluginline(fetch_config, plugin_line) + plugin, _ = prefetch_plugin(pdesc, ) if autocommit: commit( nixpkgs_repo, -- cgit 1.4.1