diff options
Diffstat (limited to 'maintainers/scripts')
| -rw-r--r-- | maintainers/scripts/all-tarballs.nix | 2 | ||||
| -rwxr-xr-x | maintainers/scripts/fix-maintainers.pl | 14 | ||||
| -rwxr-xr-x | maintainers/scripts/haskell/hydra-report.hs | 19 | ||||
| -rwxr-xr-x | maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh | 2 | ||||
| -rw-r--r-- | maintainers/scripts/luarocks-packages.csv | 27 | ||||
| -rw-r--r-- | maintainers/scripts/pluginupdate.py | 58 | ||||
| -rwxr-xr-x | maintainers/scripts/update-luarocks-packages | 216 | ||||
| -rw-r--r-- | maintainers/scripts/update-luarocks-shell.nix | 13 |
8 files changed, 80 insertions, 271 deletions
diff --git a/maintainers/scripts/all-tarballs.nix b/maintainers/scripts/all-tarballs.nix index 6a4de8a4b95..83236e6fa91 100644 --- a/maintainers/scripts/all-tarballs.nix +++ b/maintainers/scripts/all-tarballs.nix @@ -12,5 +12,5 @@ import ../../pkgs/top-level/release.nix scrubJobs = false; # No need to evaluate on i686. supportedSystems = [ "x86_64-linux" ]; - limitedSupportedSystems = []; + bootstrapConfigs = []; } diff --git a/maintainers/scripts/fix-maintainers.pl b/maintainers/scripts/fix-maintainers.pl index a83df9ec0cf..c953cff5cc4 100755 --- a/maintainers/scripts/fix-maintainers.pl +++ b/maintainers/scripts/fix-maintainers.pl @@ -13,12 +13,15 @@ STDOUT->autoflush(1); my $ua = LWP::UserAgent->new(); +if (!defined $ENV{GH_TOKEN}) { + die "Set GH_TOKEN before running this script"; +} + keys %$maintainers_json; # reset the internal iterator so a prior each() doesn't affect the loop while(my($k, $v) = each %$maintainers_json) { my $current_user = %$v{'github'}; if (!defined $current_user) { print "$k has no github handle\n"; - next; } my $github_id = %$v{'githubId'}; if (!defined $github_id) { @@ -37,13 +40,16 @@ while(my($k, $v) = each %$maintainers_json) { sleep($ratelimit_reset - time() + 5); } if ($resp->code != 200) { - print $current_user . " likely deleted their github account\n"; + print "$k likely deleted their github account\n"; next; } my $resp_json = from_json($resp->content); my $api_user = %$resp_json{"login"}; - if (lc($current_user) ne lc($api_user)) { - print $current_user . " is now known on github as " . $api_user . ". Editing maintainer-list.nix…\n"; + if (!defined $current_user) { + print "$k is known on github as $api_user.\n"; + } + elsif (lc($current_user) ne lc($api_user)) { + print "$k is now known on github as $api_user. Editing maintainer-list.nix…\n"; my $file = path($maintainers_list_nix); my $data = $file->slurp_utf8; $data =~ s/github = "$current_user";$/github = "$api_user";/m; diff --git a/maintainers/scripts/haskell/hydra-report.hs b/maintainers/scripts/haskell/hydra-report.hs index 5573e5e5afc..2ce3ecb2ae7 100755 --- a/maintainers/scripts/haskell/hydra-report.hs +++ b/maintainers/scripts/haskell/hydra-report.hs @@ -187,7 +187,7 @@ getBuildReports opt = runReq defaultHttpConfig do getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build) getEvalBuilds NoHydraSlownessWorkaround id = - hydraJSONQuery (responseTimeout 900000000) ["eval", showT id, "builds"] + hydraJSONQuery mempty ["eval", showT id, "builds"] getEvalBuilds HydraSlownessWorkaround id = do Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ] forM builds $ \buildId -> do @@ -195,14 +195,15 @@ getEvalBuilds HydraSlownessWorkaround id = do hydraJSONQuery mempty [ "build", showT buildId ] hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a) -hydraQuery responseType option query = - responseBody - <$> req - GET - (foldl' (/:) (https "hydra.nixos.org") query) - NoReqBody - responseType - (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" <> option) +hydraQuery responseType option query = do + let customHeaderOpt = + header + "User-Agent" + "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" + customTimeoutOpt = responseTimeout 900_000_000 -- 15 minutes + opts = customHeaderOpt <> customTimeoutOpt <> option + url = foldl' (/:) (https "hydra.nixos.org") query + responseBody <$> req GET url NoReqBody responseType opts hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a hydraJSONQuery = hydraQuery jsonResponse diff --git a/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh index 86fecbc3d87..9130941a536 100755 --- a/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh +++ b/maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh @@ -39,5 +39,5 @@ fi package_list="$(nix-build -A haskell.package-list)/nixos-hackage-packages.csv" username=$(grep "^username:" "$CABAL_DIR/config" | sed "s/^username: //") password_command=$(grep "^password-command:" "$CABAL_DIR/config" | sed "s/^password-command: //") -curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" http://hackage.haskell.org/distro/NixOS/packages.csv +curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" https://hackage.haskell.org/distro/NixOS/packages.csv echo diff --git a/maintainers/scripts/luarocks-packages.csv b/maintainers/scripts/luarocks-packages.csv index 52ac8a93431..78cfca24d96 100644 --- a/maintainers/scripts/luarocks-packages.csv +++ b/maintainers/scripts/luarocks-packages.csv @@ -1,9 +1,9 @@ name,src,ref,server,version,luaversion,maintainers alt-getopt,,,,,,arobyn bit32,,,,5.3.0-1,5.1,lblasc -argparse,https://github.com/luarocks/argparse.git,,,,, -basexx,https://github.com/teto/basexx.git,,,,, -binaryheap,https://github.com/Tieske/binaryheap.lua,,,,,vcunat +argparse,,,,,, +basexx,,,,,, +binaryheap,,,,,,vcunat busted,,,,,, cassowary,,,,,,marsam alerque cldr,,,,,,alerque @@ -12,8 +12,7 @@ cosmo,,,,,,marsam coxpcall,,,,1.17.0-1,, cqueues,,,,,,vcunat cyan,,,,,, -cyrussasl,https://github.com/JorjBauer/lua-cyrussasl.git,,,,, -digestif,https://github.com/astoff/digestif.git,,,0.2-1,5.3, +digestif,https://github.com/astoff/digestif.git,,,,5.3, dkjson,,,,,, fennel,,,,,,misterio77 fifo,,,,,, @@ -24,7 +23,7 @@ http,,,,0.3-0,,vcunat inspect,,,,,, jsregexp,,,,,, ldbus,,,http://luarocks.org/dev,,, -ldoc,https://github.com/stevedonovan/LDoc.git,,,,, +ldoc,,,,,, lgi,,,,,, linenoise,https://github.com/hoelzro/lua-linenoise.git,,,,, ljsyscall,,,,,5.1,lblasc @@ -34,14 +33,13 @@ loadkit,,,,,,alerque lpeg,,,,,,vyp lpeg_patterns,,,,,, lpeglabel,,,,1.6.0,, -lpty,,,,,, lrexlib-gnu,,,,,, lrexlib-pcre,,,,,,vyp lrexlib-posix,,,,,, lua-cjson,,,,,, lua-cmsgpack,,,,,, lua-curl,,,,,, -lua-iconv,,,,,, +lua-ffi-zlib,,,,,, lua-lsp,,,,,, lua-messagepack,,,,,, lua-protobuf,,,,,,lockejan @@ -50,6 +48,7 @@ lua-resty-jwt,,,,,, lua-resty-openidc,,,,,, lua-resty-openssl,,,,,, lua-resty-session,,,,,, +lua-rtoml,https://github.com/lblasc/lua-rtoml,,,,,lblasc lua-subprocess,https://github.com/0x0ade/lua-subprocess,,,,5.1,scoder12 lua-term,,,,,, lua-toml,,,,,, @@ -72,6 +71,7 @@ lualogging,,,,,, luaossl,,,,,5.1, luaposix,,,,34.1.1-1,,vyp lblasc luarepl,,,,,, +luarocks-build-rust-mlua,,,,,,mrcjkb luasec,,,,,,flosse luasocket,,,,,, luasql-sqlite3,,,,,,vyp @@ -82,31 +82,36 @@ luaunit,,,,,,lockejan luautf8,,,,,,pstn luazip,,,,,, lua-yajl,,,,,,pstn +lua-iconv,,,,7.0.0,, luuid,,,,,, luv,,,,1.44.2-1,, lush.nvim,https://github.com/rktjmp/lush.nvim,,,,,teto lyaml,,,,,,lblasc -magick,,,,,,donovanglover +magick,,,,,5.1,donovanglover markdown,,,,,, mediator_lua,,,,,, +middleclass,,,,,, mpack,,,,,, moonscript,https://github.com/leafo/moonscript.git,dev-1,,,,arobyn +nui.nvim,,,,,,mrcjkb nvim-client,https://github.com/neovim/lua-client.git,,,,, nvim-cmp,https://github.com/hrsh7th/nvim-cmp,,,,, penlight,https://github.com/lunarmodules/Penlight.git,,,,,alerque plenary.nvim,https://github.com/nvim-lua/plenary.nvim.git,,,,5.1, rapidjson,https://github.com/xpol/lua-rapidjson.git,,,,, rest.nvim,,,,,5.1,teto -readline,,,,,, +rustaceanvim,,,,,,mrcjkb say,https://github.com/Olivine-Labs/say.git,,,,, serpent,,,,,,lockejan sqlite,,,,,, std._debug,https://github.com/lua-stdlib/_debug.git,,,,, -std.normalize,https://github.com/lua-stdlib/normalize.git,,,,, +std.normalize,,,,,, stdlib,,,,41.2.2,,vyp teal-language-server,,,http://luarocks.org/dev,,, telescope.nvim,,,,,5.1, telescope-manix,,,,,, tl,,,,,,mephistophiles +toml,,,,,,mrcjkb +toml-edit,,,,,5.1,mrcjkb vstruct,https://github.com/ToxicFrog/vstruct.git,,,,, vusted,,,,,,figsoda diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py index 6a607eb6248..cc0f4ef742d 100644 --- a/maintainers/scripts/pluginupdate.py +++ b/maintainers/scripts/pluginupdate.py @@ -26,7 +26,7 @@ import urllib.parse import urllib.request import xml.etree.ElementTree as ET from dataclasses import asdict, dataclass -from datetime import datetime +from datetime import UTC, datetime from functools import wraps from multiprocessing.dummy import Pool from pathlib import Path @@ -321,8 +321,13 @@ def load_plugins_from_csv( return plugins -def run_nix_expr(expr): - with CleanEnvironment() as nix_path: + +def run_nix_expr(expr, nixpkgs: str): + ''' + :param expr nix expression to fetch current plugins + :param nixpkgs Path towards a nixpkgs checkout + ''' + with CleanEnvironment(nixpkgs) as nix_path: cmd = [ "nix", "eval", @@ -335,8 +340,8 @@ def run_nix_expr(expr): "--nix-path", nix_path, ] - log.debug("Running command %s", " ".join(cmd)) - out = subprocess.check_output(cmd) + log.debug("Running command: %s", " ".join(cmd)) + out = subprocess.check_output(cmd, timeout=90) data = json.loads(out) return data @@ -396,9 +401,9 @@ class Editor: """CSV spec""" print("the update member function should be overriden in subclasses") - def get_current_plugins(self) -> List[Plugin]: + def get_current_plugins(self, nixpkgs) -> List[Plugin]: """To fill the cache""" - data = run_nix_expr(self.get_plugins) + data = run_nix_expr(self.get_plugins, nixpkgs) plugins = [] for name, attr in data.items(): p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"]) @@ -414,7 +419,7 @@ class Editor: raise NotImplementedError() def get_update(self, input_file: str, outfile: str, config: FetchConfig): - cache: Cache = Cache(self.get_current_plugins(), self.cache_file) + cache: Cache = Cache(self.get_current_plugins(self.nixpkgs), self.cache_file) _prefetch = functools.partial(prefetch, cache=cache) def update() -> dict: @@ -454,9 +459,16 @@ class Editor: ), ) common.add_argument( + "--nixpkgs", + type=str, + default=os.getcwd(), + help="Adjust log level", + ) + common.add_argument( "--input-names", "-i", dest="input_file", + type=Path, default=self.default_in, help="A list of plugins in the form owner/repo", ) @@ -465,6 +477,7 @@ class Editor: "-o", dest="outfile", default=self.default_out, + type=Path, help="Filename to save generated nix code", ) common.add_argument( @@ -541,22 +554,26 @@ class Editor: command = args.command or "update" log.setLevel(LOG_LEVELS[args.debug]) log.info("Chose to run command: %s", command) + self.nixpkgs = args.nixpkgs - if not args.no_commit: - self.nixpkgs_repo = git.Repo(self.root, search_parent_directories=True) + self.nixpkgs_repo = git.Repo(args.nixpkgs, search_parent_directories=True) getattr(self, command)(args) class CleanEnvironment(object): + def __init__(self, nixpkgs): + self.local_pkgs = nixpkgs + def __enter__(self) -> str: - self.old_environ = os.environ.copy() + """ local_pkgs = str(Path(__file__).parent.parent.parent) + """ + self.old_environ = os.environ.copy() self.empty_config = NamedTemporaryFile() self.empty_config.write(b"{}") self.empty_config.flush() - os.environ["NIXPKGS_CONFIG"] = self.empty_config.name - return f"localpkgs={local_pkgs}" + return f"localpkgs={self.local_pkgs}" def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: os.environ.update(self.old_environ) @@ -758,7 +775,8 @@ def commit(repo: git.Repo, message: str, files: List[Path]) -> None: def update_plugins(editor: Editor, args): - """The main entry function of this module. All input arguments are grouped in the `Editor`.""" + """The main entry function of this module. + All input arguments are grouped in the `Editor`.""" log.info("Start updating plugins") fetch_config = FetchConfig(args.proc, args.github_token) @@ -770,8 +788,16 @@ def update_plugins(editor: Editor, args): autocommit = not args.no_commit if autocommit: - editor.nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True) - commit(editor.nixpkgs_repo, f"{editor.attr_path}: update", [args.outfile]) + try: + repo = git.Repo(os.getcwd()) + updated = datetime.now(tz=UTC).strftime('%Y-%m-%d') + print(args.outfile) + commit(repo, + f"{editor.attr_path}: update on {updated}", [args.outfile] + ) + except git.InvalidGitRepositoryError as e: + print(f"Not in a git repository: {e}", file=sys.stderr) + sys.exit(1) if redirects: update() diff --git a/maintainers/scripts/update-luarocks-packages b/maintainers/scripts/update-luarocks-packages deleted file mode 100755 index 791cd8a1d89..00000000000 --- a/maintainers/scripts/update-luarocks-packages +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell update-luarocks-shell.nix -i python3 - -# format: -# $ nix run nixpkgs.python3Packages.black -c black update.py -# type-check: -# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py -# linted: -# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265,E402 update.py - -import inspect -import os -import tempfile -import shutil -from dataclasses import dataclass -import subprocess -import csv -import logging -import textwrap -from multiprocessing.dummy import Pool - -from typing import List, Tuple, Optional -from pathlib import Path - -log = logging.getLogger() -log.addHandler(logging.StreamHandler()) - -ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent # type: ignore -import pluginupdate -from pluginupdate import update_plugins, FetchConfig, CleanEnvironment - -PKG_LIST="maintainers/scripts/luarocks-packages.csv" -TMP_FILE="$(mktemp)" -GENERATED_NIXFILE="pkgs/development/lua-modules/generated-packages.nix" -LUAROCKS_CONFIG="maintainers/scripts/luarocks-config.lua" - -HEADER = """/* {GENERATED_NIXFILE} is an auto-generated file -- DO NOT EDIT! -Regenerate it with: -nixpkgs$ ./maintainers/scripts/update-luarocks-packages - -You can customize the generated packages in pkgs/development/lua-modules/overrides.nix -*/ -""".format(GENERATED_NIXFILE=GENERATED_NIXFILE) - -FOOTER=""" -} -/* GENERATED - do not edit this file */ -""" - -@dataclass -class LuaPlugin: - name: str - '''Name of the plugin, as seen on luarocks.org''' - src: str - '''address to the git repository''' - ref: Optional[str] - '''git reference (branch name/tag)''' - version: Optional[str] - '''Set it to pin a package ''' - server: Optional[str] - '''luarocks.org registers packages under different manifests. - Its value can be 'http://luarocks.org/dev' - ''' - luaversion: Optional[str] - '''Attribue of the lua interpreter if a package is available only for a specific lua version''' - maintainers: Optional[str] - ''' Optional string listing maintainers separated by spaces''' - - @property - def normalized_name(self) -> str: - return self.name.replace(".", "-") - -# rename Editor to LangUpdate/ EcosystemUpdater -class LuaEditor(pluginupdate.Editor): - def get_current_plugins(self): - return [] - - def load_plugin_spec(self, input_file) -> List[LuaPlugin]: - luaPackages = [] - csvfilename=input_file - log.info("Loading package descriptions from %s", csvfilename) - - with open(csvfilename, newline='') as csvfile: - reader = csv.DictReader(csvfile,) - for row in reader: - # name,server,version,luaversion,maintainers - plugin = LuaPlugin(**row) - luaPackages.append(plugin) - return luaPackages - - def update(self, args): - update_plugins(self, args) - - def generate_nix( - self, - results: List[Tuple[LuaPlugin, str]], - outfilename: str - ): - - with tempfile.NamedTemporaryFile("w+") as f: - f.write(HEADER) - header2 = textwrap.dedent( - # header2 = inspect.cleandoc( - """ - { self, stdenv, lib, fetchurl, fetchgit, callPackage, ... } @ args: - final: prev: - { - """) - f.write(header2) - for (plugin, nix_expr) in results: - f.write(f"{plugin.normalized_name} = {nix_expr}") - f.write(FOOTER) - f.flush() - - # if everything went fine, move the generated file to its destination - # using copy since move doesn't work across disks - shutil.copy(f.name, outfilename) - - print(f"updated {outfilename}") - - @property - def attr_path(self): - return "luaPackages" - - def get_update(self, input_file: str, outfile: str, config: FetchConfig): - _prefetch = generate_pkg_nix - - def update() -> dict: - plugin_specs = self.load_plugin_spec(input_file) - sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower()) - - try: - pool = Pool(processes=config.proc) - results = pool.map(_prefetch, sorted_plugin_specs) - finally: - pass - - self.generate_nix(results, outfile) - - redirects = {} - return redirects - - return update - - def rewrite_input(self, input_file: str, *args, **kwargs): - # vim plugin reads the file before update but that shouldn't be our case - # not implemented yet - # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers'] - # input_file = "toto.csv" - # with open(input_file, newline='') as csvfile: - # writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - # writer.writeheader() - # for row in reader: - # # name,server,version,luaversion,maintainers - # plugin = LuaPlugin(**row) - # luaPackages.append(plugin) - pass - -def generate_pkg_nix(plug: LuaPlugin): - ''' - Generate nix expression for a luarocks package - Our cache key associates "p.name-p.version" to its rockspec - ''' - log.debug("Generating nix expression for %s", plug.name) - custom_env = os.environ.copy() - custom_env['LUAROCKS_CONFIG'] = LUAROCKS_CONFIG - - # we add --dev else luarocks wont find all the "scm" (=dev) versions of the - # packages - # , "--dev" - cmd = [ "luarocks", "nix" ] - - if plug.maintainers: - cmd.append(f"--maintainers={plug.maintainers}") - - # if plug.server == "src": - if plug.src != "": - if plug.src is None: - msg = "src must be set when 'version' is set to \"src\" for package %s" % plug.name - log.error(msg) - raise RuntimeError(msg) - log.debug("Updating from source %s", plug.src) - cmd.append(plug.src) - # update the plugin from luarocks - else: - cmd.append(plug.name) - if plug.version and plug.version != "src": - - cmd.append(plug.version) - - if plug.server != "src" and plug.server: - cmd.append(f"--only-server={plug.server}") - - if plug.luaversion: - cmd.append(f"--lua-version={plug.luaversion}") - - log.debug("running %s", ' '.join(cmd)) - - output = subprocess.check_output(cmd, env=custom_env, text=True) - output = "callPackage(" + output.strip() + ") {};\n\n" - return (plug, output) - -def main(): - - editor = LuaEditor("lua", ROOT, '', - default_in = ROOT.joinpath(PKG_LIST), - default_out = ROOT.joinpath(GENERATED_NIXFILE) - ) - - editor.run() - -if __name__ == "__main__": - - main() - -# vim: set ft=python noet fdm=manual fenc=utf-8 ff=unix sts=0 sw=4 ts=4 : diff --git a/maintainers/scripts/update-luarocks-shell.nix b/maintainers/scripts/update-luarocks-shell.nix deleted file mode 100644 index 346b0319b08..00000000000 --- a/maintainers/scripts/update-luarocks-shell.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ nixpkgs ? import ../.. { } -}: -with nixpkgs; -let - pyEnv = python3.withPackages(ps: [ ps.gitpython ]); -in -mkShell { - packages = [ - pyEnv - luarocks-nix - nix-prefetch-scripts - ]; -} |