linux: rewrite updater-script, make data-driven

Co-authored-by: Raito Bezarius <masterancpp@gmail.com>

14 files changed, 189 insertions, 240 deletions

diff --git a/pkgs/os-specific/linux/kernel/kernels-org.json b/pkgs/os-specific/linux/kernel/kernels-org.json
new file mode 100644
index 00000000000..b2d4b1fa83b
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/kernels-org.json
@@ -0,0 +1,38 @@
+{
+    "testing": {
+        "version": "6.6-rc1",
+        "hash": "02zh3dnikyhhlas9xccia963d4yqmzq0m4b8s10x8mjng3na45hd"
+    },
+    "6.5": {
+        "version": "6.5.4",
+        "hash": "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx"
+    },
+    "6.4": {
+        "version": "6.4.16",
+        "hash": "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln"
+    },
+    "6.1": {
+        "version": "6.1.54",
+        "hash": "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653"
+    },
+    "5.15": {
+        "version": "5.15.132",
+        "hash": "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1"
+    },
+    "5.10": {
+        "version": "5.10.195",
+        "hash": "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1"
+    },
+    "5.4": {
+        "version": "5.4.256",
+        "hash": "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967"
+    },
+    "4.19": {
+        "version": "4.19.294",
+        "hash": "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc"
+    },
+    "4.14": {
+        "version": "4.14.325",
+        "hash": "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav"
+    }
+}
diff --git a/pkgs/os-specific/linux/kernel/linux-4.14.nix b/pkgs/os-specific/linux/kernel/linux-4.14.nix
deleted file mode 100644
index dc0ead44be4..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-4.14.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "4.14.325";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
-    sha256 = "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-4.19.nix b/pkgs/os-specific/linux/kernel/linux-4.19.nix
deleted file mode 100644
index 6048caf8fa8..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-4.19.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "4.19.294";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v4.x/linux-${version}.tar.xz";
-    sha256 = "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.10.nix b/pkgs/os-specific/linux/kernel/linux-5.10.nix
deleted file mode 100644
index 213bf7e670f..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.10.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.10.195";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.15.nix b/pkgs/os-specific/linux/kernel/linux-5.15.nix
deleted file mode 100644
index d86b0cf0ce6..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.15.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.15.132";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-5.4.nix b/pkgs/os-specific/linux/kernel/linux-5.4.nix
deleted file mode 100644
index 0013636076a..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-5.4.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "5.4.256";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-    sha256 = "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967";
-  };
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.1.nix b/pkgs/os-specific/linux/kernel/linux-6.1.nix
deleted file mode 100644
index df99c986851..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.1.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, buildPackages, fetchurl, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.1.54";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.4.nix b/pkgs/os-specific/linux/kernel/linux-6.4.nix
deleted file mode 100644
index d41cd21fbef..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.4.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, fetchurl, buildLinux, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.4.16";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-6.5.nix b/pkgs/os-specific/linux/kernel/linux-6.5.nix
deleted file mode 100644
index 28557090de6..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-6.5.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ lib, fetchurl, buildLinux, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.5.4";
-
-  # modDirVersion needs to be x.y.z, will automatically add .0 if needed
-  modDirVersion = versions.pad 3 version;
-
-  # branchVersion needs to be x.y
-  extraMeta.branch = versions.majorMinor version;
-
-  src = fetchurl {
-    url = "mirror://kernel/linux/kernel/v6.x/linux-${version}.tar.xz";
-    sha256 = "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx";
-  };
-} // (args.argsOverride or { }))
diff --git a/pkgs/os-specific/linux/kernel/linux-testing.nix b/pkgs/os-specific/linux/kernel/linux-testing.nix
deleted file mode 100644
index 9a3b32a7f2d..00000000000
--- a/pkgs/os-specific/linux/kernel/linux-testing.nix
+++ /dev/null
@@ -1,20 +0,0 @@
-{ lib, buildPackages, fetchzip, perl, buildLinux, nixosTests, ... } @ args:
-
-with lib;
-
-buildLinux (args // rec {
-  version = "6.6-rc1";
-  extraMeta.branch = lib.versions.majorMinor version;
-
-  # modDirVersion needs to be x.y.z, will always add .0
-  modDirVersion = versions.pad 3 version;
-
-  src = fetchzip {
-    url = "https://git.kernel.org/torvalds/t/linux-${version}.tar.gz";
-    hash = "sha256-DRai7HhWVtRB0GiRCvCv2JM2TFKRsZ60ohD6GW0b8As=";
-  };
-
-  # Should the testing kernels ever be built on Hydra?
-  extraMeta.hydraPlatforms = [];
-
-} // (args.argsOverride or {}))
diff --git a/pkgs/os-specific/linux/kernel/mainline.nix b/pkgs/os-specific/linux/kernel/mainline.nix
new file mode 100644
index 00000000000..50053e620e4
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/mainline.nix
@@ -0,0 +1,18 @@
+{ branch, lib, fetchurl, buildLinux, ... } @ args:
+
+let
+  allKernels = builtins.fromJSON (builtins.readFile ./kernels-org.json);
+  thisKernel = allKernels.${branch};
+
+  args' = (builtins.removeAttrs args ["branch"]) // rec {
+    inherit (thisKernel) version;
+    modDirVersion = lib.versions.pad 3 version;
+    extraMeta.branch = branch;
+
+    src = fetchurl {
+      url = "mirror://kernel/linux/kernel/v${lib.versions.major version}.x/linux-${version}.tar.xz";
+      sha256 = thisKernel.hash;
+    };
+  } // (args.argsOverride or {});
+in
+buildLinux args'
diff --git a/pkgs/os-specific/linux/kernel/update-mainline.py b/pkgs/os-specific/linux/kernel/update-mainline.py
new file mode 100755
index 00000000000..e7c37e9ab99
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel/update-mainline.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages (ps: [ ps.beautifulsoup4 ps.lxml ])"
+from enum import Enum
+from bs4 import BeautifulSoup, NavigableString, Tag
+from dataclasses import dataclass
+import json
+import pathlib
+import re
+import subprocess
+import urllib.request
+import sys
+
+
+HERE = pathlib.Path(__file__).parent
+ROOT = HERE.parent.parent.parent.parent
+VERSIONS_FILE = HERE / "kernels-org.json"
+
+class KernelNature(Enum):
+    MAINLINE = 1
+    STABLE = 2
+    LONGTERM = 3
+
+@dataclass
+class KernelRelease:
+    nature: KernelNature
+    version: str
+    date: str
+    link: str
+    eol: bool = False
+
+def parse_release(release: Tag) -> KernelRelease | None:
+    columns: list[Tag] = list(release.find_all('td'))
+    try:
+        nature = KernelNature[columns[0].get_text().rstrip(':').upper()]
+    except KeyError:
+        return None
+
+    version = columns[1].get_text().rstrip(' [EOL]')
+    date = columns[2].get_text()
+    link = columns[3].find('a')
+    if link is not None and isinstance(link, Tag):
+        link = link.attrs.get('href')
+    assert link is not None, f'link for kernel {version} is non-existent'
+    eol = bool(release.find(class_='eolkernel'))
+
+    return KernelRelease(nature=nature, version=version, date=date, link=link, eol=eol)
+
+def get_branch(version: str):
+    # This is a testing kernel.
+    if 'rc' in version:
+        return 'testing'
+    else:
+        major, minor, *_ = version.split(".")
+        return f"{major}.{minor}"
+
+
+def get_hash(url: str):
+    return subprocess.check_output(["nix-prefetch-url", url]).decode().strip()
+
+
+def commit(message):
+    return subprocess.check_call(["git", "commit", "-m", message, VERSIONS_FILE])
+
+
+def main():
+    kernel_org = urllib.request.urlopen("https://kernel.org/")
+    soup = BeautifulSoup(kernel_org.read().decode(), "lxml")
+    release_table = soup.find(id='releases')
+    if not release_table or isinstance(release_table, NavigableString):
+        print(release_table)
+        print('Failed to find the release table on https://kernel.org')
+        sys.exit(1)
+
+    releases = release_table.find_all('tr')
+    parsed_releases = filter(None, [parse_release(release) for release in releases])
+    all_kernels = json.load(VERSIONS_FILE.open())
+
+    for kernel in parsed_releases:
+        branch = get_branch(kernel.version)
+        nixpkgs_branch = branch.replace('.', '_')
+
+        old_version = all_kernels.get(branch, {}).get("version")
+        if old_version == kernel.version:
+            print(f"linux_{nixpkgs_branch}: {kernel.version} is latest, skipping...")
+            continue
+
+        if old_version is None:
+            message = f"linux_{nixpkgs_branch}: init at {kernel.version}"
+        else:
+            message = f"linux_{nixpkgs_branch}: {old_version} -> {kernel.version}"
+
+        print(message)
+
+        all_kernels[branch] = {"version": kernel.version, "hash": get_hash(kernel.link)}
+
+        with VERSIONS_FILE.open("w") as fd:
+            json.dump(all_kernels, fd, indent=4)
+            fd.write("\n")  # makes editorconfig happy
+
+        commit(message)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/pkgs/os-specific/linux/kernel/update.sh b/pkgs/os-specific/linux/kernel/update.sh
index 4171b7492b7..37e1cc1a5cd 100755
--- a/pkgs/os-specific/linux/kernel/update.sh
+++ b/pkgs/os-specific/linux/kernel/update.sh
@@ -1,72 +1,14 @@
 #!/usr/bin/env bash
-set -e
+cd "$(dirname "$(readlink -f "$0")")" || exit
 
-# Get the latest versions from kernel.org
-LINUXSED='s/.*linux-\([0-9]\+\(.[0-9]\+\)*\).*/\1/p'
-KDATA="$(curl -s https://www.kernel.org | sed -n -e '/Download complete/p')"
-VERSIONS=($(sed -n -e $LINUXSED <<< "$KDATA" | sort -Vr))
+echo "Update linux (mainline)"
+COMMIT=1 ./update-mainline.py || echo "update-mainline failed with exit code $?"
 
-# Remove mainline version if there is a stable update
-# Note due to sorting these two will always exist at the bottom
-if grep -q "^${VERSIONS[1]}" <<< "${VERSIONS[0]}"; then
-  VERSIONS=(${VERSIONS[@]:0:1} ${VERSIONS[@]:2})
-fi
+echo "Update linux-rt"
+COMMIT=1 ./update-rt.sh || echo "update-rt failed with exit code $?"
 
-# Inspect each file and see if it has the latest version
-NIXPKGS="$(git rev-parse --show-toplevel)"
-ls $NIXPKGS/pkgs/os-specific/linux/kernel | while read FILE; do
-  KERNEL="$(sed -n -e $LINUXSED <<< "$FILE")"
-  [ -z "$KERNEL" ] && continue
+echo "Update linux-libre"
+COMMIT=1 ./update-libre.sh || echo "update-libre failed with exit code $?"
 
-  # Find the matching new kernel version
-  MATCHING=""
-  for V in "${VERSIONS[@]}"; do
-    if grep -q "^$KERNEL" <<< "$V"; then
-      MATCHING="$V"
-      break
-    fi
-  done
-  if [ -z "$MATCHING" ]; then
-    echo "Out-of-support $KERNEL"
-    continue
-  fi
-
-  # Inspect the nix expression to check for changes
-  DATA="$(<$NIXPKGS/pkgs/os-specific/linux/kernel/$FILE)"
-  URL="$(sed -n -e 's/.*url = "\(.*\)";.*/\1/p' <<< "$DATA" | sed -e "s/\${version}/$MATCHING/g")"
-  OLDVER=$(sed -n -e 's/.*version = "\(.*\)".*/\1/p' <<< "$DATA")
-  if [ "$OLDVER" = "$V" ]; then
-    echo "No updates for $KERNEL"
-    continue
-  fi
-
-  # Download the new file for the hash
-  if ! HASH="$(nix-prefetch-url $URL 2>/dev/null)"; then
-    echo "Failed to get hash of $URL"
-    continue
-  fi
-  sed -i -e "s/sha256 = \".*\"/sha256 = \"$HASH\"/g" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-
-  # Rewrite the expression
-  sed -i -e '/version = /d' $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-  sed -i -e "\#buildLinux (args // rec {#a \  version = \"$V\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-
-  # Commit the changes
-  git add -u $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
-  git commit -m "linux: $OLDVER -> $V" >/dev/null 2>&1
-
-  echo "Updated $OLDVER -> $V"
-done
-
-# Allowing errors again: one broken update script shouldn't inhibit the
-# update of other kernel variants.
-set +e
-
-echo Update linux-rt
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-rt.sh || echo "update-rt failed with exit code $?"
-
-echo Update linux-libre
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh || echo "update-libre failed with exit code $?"
-
-echo Update linux-hardened
-COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py || echo "update-hardened failed with exit code $?"
+echo "Update linux-hardened"
+COMMIT=1 ./hardened/update.py || echo "update-hardened failed with exit code $?"
diff --git a/pkgs/top-level/linux-kernels.nix b/pkgs/top-level/linux-kernels.nix
index 2ba4a61e8b7..7b746286f05 100644
--- a/pkgs/top-level/linux-kernels.nix
+++ b/pkgs/top-level/linux-kernels.nix
@@ -96,7 +96,8 @@ in {
       rpiVersion = 4;
     };
 
-    linux_4_14 = callPackage ../os-specific/linux/kernel/linux-4.14.nix {
+    linux_4_14 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "4.14";
       kernelPatches =
         [ kernelPatches.bridge_stp_helper
           kernelPatches.request_key_helper
@@ -107,7 +108,8 @@ in {
         ];
     };
 
-    linux_4_19 = callPackage ../os-specific/linux/kernel/linux-4.19.nix {
+    linux_4_19 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "4.19";
       kernelPatches =
         [ kernelPatches.bridge_stp_helper
           kernelPatches.request_key_helper
@@ -115,7 +117,8 @@ in {
         ];
     };
 
-    linux_5_4 = callPackage ../os-specific/linux/kernel/linux-5.4.nix {
+    linux_5_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.4";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -130,7 +133,8 @@ in {
       ];
     };
 
-    linux_5_10 = callPackage ../os-specific/linux/kernel/linux-5.10.nix {
+    linux_5_10 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.10";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -145,7 +149,8 @@ in {
       ];
     };
 
-    linux_5_15 = callPackage ../os-specific/linux/kernel/linux-5.15.nix {
+    linux_5_15 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "5.15";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -160,7 +165,8 @@ in {
       ];
     };
 
-    linux_6_1 = callPackage ../os-specific/linux/kernel/linux-6.1.nix {
+    linux_6_1 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.1";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -177,7 +183,8 @@ in {
       ];
     };
 
-    linux_6_4 = callPackage ../os-specific/linux/kernel/linux-6.4.nix {
+    linux_6_4 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.4";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -185,7 +192,8 @@ in {
       ];
     };
 
-    linux_6_5 = callPackage ../os-specific/linux/kernel/linux-6.5.nix {
+    linux_6_5 = callPackage ../os-specific/linux/kernel/mainline.nix {
+      branch = "6.5";
       kernelPatches = [
         kernelPatches.bridge_stp_helper
         kernelPatches.request_key_helper
@@ -194,7 +202,10 @@ in {
     };
 
     linux_testing = let
-      testing = callPackage ../os-specific/linux/kernel/linux-testing.nix {
+      testing = callPackage ../os-specific/linux/kernel/mainline.nix {
+        # A special branch that tracks the kernel under the release process
+        # i.e. which has at least a public rc1 and is not released yet.
+        branch = "testing";
         kernelPatches = [
           kernelPatches.bridge_stp_helper
           kernelPatches.request_key_helper