patches and low-level development discussion
 help / color / mirror / code / Atom feed
* [PATCH 1/2] chromiumOSPackages: improve update.py repo compat
@ 2020-05-30 19:00 Alyssa Ross
  2020-05-30 19:00 ` [PATCH 2/2] chromiumOSPackages: 81.12871.0.0-rc1 -> 83.13020.0.0-rc1 Alyssa Ross
                   ` (2 more replies)
  0 siblings, 3 replies; 16+ messages in thread
From: Alyssa Ross @ 2020-05-30 19:00 UTC (permalink / raw)
  To: devel; +Cc: Cole Helbling, Alyssa Ross

These changes make chromiumOSPackages.updateScript better understand
the Repo manifest format[1].

It now properly distinguishes between paths in the Chromium OS source
tree and paths on the git servers, which are usually not the same.
For example, chromiumos-overlay is located in the source tree at
src/third_party/chromiumos-overlay, but on chromium.googlesource.com
it's located at chromiumos/overlays/chromiumos-overlays.  Components
are now keyed by their location in the Chromium OS source tree (and
packages have all been updated for this change.)

Additionally, it now understands Repo remotes.  This means that it is
now possible to use a Chromium OS component like Minijail, which is
hosted on android.googlesource.com rather than
chromiumos.googlesource.com.  This was not previously possible.

[1]: https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
---
 .../linux/chromium-os/common-mk/default.nix   |  2 +-
 .../linux/chromium-os/crosvm/default.nix      |  4 +-
 .../os-specific/linux/chromium-os/default.nix |  2 +-
 .../linux/chromium-os/libqmi/default.nix      |  2 +-
 .../linux/chromium-os/minigbm/default.nix     |  2 +-
 .../linux/chromium-os/modem-manager/next.nix  |  2 +-
 .../linux/chromium-os/modp_b64/default.nix    |  2 +-
 pkgs/os-specific/linux/chromium-os/update.py  | 56 ++++++++++++-------
 .../linux/chromium-os/upstream-info.json      | 40 ++++++-------
 pkgs/os-specific/linux/kernel/linux-cros.nix  |  2 +-
 10 files changed, 64 insertions(+), 50 deletions(-)

diff --git a/pkgs/os-specific/linux/chromium-os/common-mk/default.nix b/pkgs/os-specific/linux/chromium-os/common-mk/default.nix
index f07e418b56f..73862bcaa97 100644
--- a/pkgs/os-specific/linux/chromium-os/common-mk/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/common-mk/default.nix
@@ -24,7 +24,7 @@
 , ... } @ args:
 
 let
-  platform2 = fetchFromGitiles upstreamInfo.components."chromiumos/platform2";
+  platform2 = fetchFromGitiles upstreamInfo.components."src/platform2";
 
   attrsToGnList = lib.mapAttrsToList (name: value: "${name}=${toGn value}");
 
diff --git a/pkgs/os-specific/linux/chromium-os/crosvm/default.nix b/pkgs/os-specific/linux/chromium-os/crosvm/default.nix
index 3813e3eb75a..8698ef67a83 100644
--- a/pkgs/os-specific/linux/chromium-os/crosvm/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/crosvm/default.nix
@@ -9,10 +9,10 @@ let
     else throw "no seccomp policy files available for host platform";
 
   crosvmSrc = fetchFromGitiles
-    upstreamInfo.components."chromiumos/platform/crosvm";
+    upstreamInfo.components."src/platform/crosvm";
 
   adhdSrc = fetchFromGitiles
-    upstreamInfo.components."chromiumos/third_party/adhd";
+    upstreamInfo.components."src/third_party/adhd";
 in
 
   rustPlatform.buildRustPackage rec {
diff --git a/pkgs/os-specific/linux/chromium-os/default.nix b/pkgs/os-specific/linux/chromium-os/default.nix
index f9c6c88d072..db719e8b0f1 100644
--- a/pkgs/os-specific/linux/chromium-os/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/default.nix
@@ -9,7 +9,7 @@ let
     upstreamInfo = lib.importJSON ./upstream-info.json;
 
     chromiumos-overlay = (fetchFromGitiles
-      upstreamInfo.components."chromiumos/overlays/chromiumos-overlay") // {
+      upstreamInfo.components."src/third_party/chromiumos-overlay") // {
         passthru.updateScript = ./update.py;
       };
 
diff --git a/pkgs/os-specific/linux/chromium-os/libqmi/default.nix b/pkgs/os-specific/linux/chromium-os/libqmi/default.nix
index ac444848b81..ec4f44c7047 100644
--- a/pkgs/os-specific/linux/chromium-os/libqmi/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/libqmi/default.nix
@@ -8,7 +8,7 @@ libqmi.overrideAttrs (
     pname = "libqmi-unstable";
     version = "2019-12-16";
 
-    src = fetchFromGitiles upstreamInfo.components."chromiumos/third_party/libqmi";
+    src = fetchFromGitiles upstreamInfo.components."src/third_party/libqmi";
 
     nativeBuildInputs = nativeBuildInputs ++
       [ autoreconfHook autoconf-archive gtk-doc docbook-xsl-nons ];
diff --git a/pkgs/os-specific/linux/chromium-os/minigbm/default.nix b/pkgs/os-specific/linux/chromium-os/minigbm/default.nix
index 0a3cbf15997..6c532a4deb2 100644
--- a/pkgs/os-specific/linux/chromium-os/minigbm/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/minigbm/default.nix
@@ -4,7 +4,7 @@ stdenv.mkDerivation {
   name = "minigbm";
   inherit (upstreamInfo) version;
 
-  src = fetchFromGitiles upstreamInfo.components."chromiumos/platform/minigbm";
+  src = fetchFromGitiles upstreamInfo.components."src/platform/minigbm";
 
   nativeBuildInputs = [ pkg-config ];
   buildInputs = [ libdrm ];
diff --git a/pkgs/os-specific/linux/chromium-os/modem-manager/next.nix b/pkgs/os-specific/linux/chromium-os/modem-manager/next.nix
index a074e82ff4b..91aa18b3846 100644
--- a/pkgs/os-specific/linux/chromium-os/modem-manager/next.nix
+++ b/pkgs/os-specific/linux/chromium-os/modem-manager/next.nix
@@ -9,7 +9,7 @@ modemmanager.overrideAttrs (
     version = "2019-10-17";
 
     src = fetchFromGitiles
-      upstreamInfo.components."chromiumos/third_party/modemmanager-next";
+      upstreamInfo.components."src/third_party/modemmanager-next";
 
     nativeBuildInputs = nativeBuildInputs ++ [ autoreconfHook libqmi libxslt ];
 
diff --git a/pkgs/os-specific/linux/chromium-os/modp_b64/default.nix b/pkgs/os-specific/linux/chromium-os/modp_b64/default.nix
index 50d91479193..557769336ce 100644
--- a/pkgs/os-specific/linux/chromium-os/modp_b64/default.nix
+++ b/pkgs/os-specific/linux/chromium-os/modp_b64/default.nix
@@ -3,7 +3,7 @@
 common-mk {
   platformSubdir = "modp_b64";
 
-  src = fetchFromGitiles upstreamInfo.components."aosp/platform/external/modp_b64";
+  src = fetchFromGitiles upstreamInfo.components."src/third_party/modp_b64";
 
   installPhase = ''
     mkdir -p $out/lib
diff --git a/pkgs/os-specific/linux/chromium-os/update.py b/pkgs/os-specific/linux/chromium-os/update.py
index 1405c534d8b..58a15dbb8e8 100755
--- a/pkgs/os-specific/linux/chromium-os/update.py
+++ b/pkgs/os-specific/linux/chromium-os/update.py
@@ -13,16 +13,16 @@ from re import MULTILINE, fullmatch, match, search
 from urllib.request import urlopen
 
 # ChromiumOS components used in Nixpkgs
-components = [
-    'aosp/platform/external/modp_b64',
-    'chromiumos/overlays/chromiumos-overlay',
-    'chromiumos/platform/crosvm',
-    'chromiumos/platform/minigbm',
-    'chromiumos/platform2',
-    'chromiumos/third_party/adhd',
-    'chromiumos/third_party/kernel',
-    'chromiumos/third_party/libqmi',
-    'chromiumos/third_party/modemmanager-next',
+component_paths = [
+    'src/platform/crosvm',
+    'src/platform/minigbm',
+    'src/platform2',
+    'src/third_party/adhd',
+    'src/third_party/chromiumos-overlay',
+    'src/third_party/kernel/v5.4',
+    'src/third_party/libqmi',
+    'src/third_party/modemmanager-next',
+    'src/third_party/modp_b64',
 ]
 
 git_root = 'https://chromium.googlesource.com/'
@@ -65,14 +65,28 @@ with urlopen(f'{buildspecs_url}{chrome_major_version}/?format=TEXT') as resp:
     buildspecs.sort(reverse=True)
     buildspec = splitext(buildspecs[0])[0]
 
-revisions = {}
+components = {}
 
 # Read the buildspec, and extract the git revisions for each component.
 with urlopen(f'{buildspecs_url}{chrome_major_version}/{buildspec}.xml?format=TEXT') as resp:
     xml = base64.decodebytes(resp.read())
     root = etree.fromstring(xml)
+
+    default_remote = root.find('default').get('remote')
+    remotes = {}
+    for remote in root.findall('remote'):
+        remotes[remote.get('name')] = remote.get('fetch')
+
     for project in root.findall('project'):
-        revisions[project.get('name')] = project.get('revision')
+        name = project.get('name')
+        path = project.get('path')
+        remote_name = project.get('remote') or default_remote
+        remote = remotes[remote_name]
+
+        components[path] = {
+            'revision': project.get('revision'),
+            'url': f'{remote}/{name}',
+        }
 
 # Initialize the data that will be output from this script.  Leave the
 # rc number in buildspec so nobody else is subject to the same level
@@ -84,15 +98,15 @@ paths = {}
 # Fill in the 'components' dictionary with the output from
 # nix-prefetch-git, which can be passed straight to fetchGit when
 # imported by Nix.
-for component in components:
-    name = component.split('/')[-1]
-    url = f'{git_root}{component}'
-    rev = revisions[component]
+for path in component_paths:
+    name = path.split('/')[-1]
+    url = components[path]['url']
+    rev = components[path]['revision']
     tarball = f'{url}/+archive/{rev}.tar.gz'
     output = subprocess.check_output(['nix-prefetch-url', '--print-path', '--unpack', '--name', name, tarball])
-    (sha256, path) = output.decode('utf-8').splitlines()
-    paths[component] = path
-    data['components'][component] = {
+    (sha256, store_path) = output.decode('utf-8').splitlines()
+    paths[path] = store_path
+    data['components'][path] = {
         'name': name,
         'url': url,
         'rev': rev,
@@ -100,14 +114,14 @@ for component in components:
     }
 
 # Get the version number of the kernel.
-kernel = paths['chromiumos/third_party/kernel']
+kernel = paths['src/third_party/kernel/v5.4']
 makefile = open(f'{kernel}/Makefile').read()
 version = search(r'^VERSION = (.+)$', makefile, MULTILINE)[1]
 patchlevel = search(r'^PATCHLEVEL = (.*?)$', makefile, MULTILINE)[1]
 sublevel = search(r'^SUBLEVEL = (.*?)$', makefile, MULTILINE)[1]
 extra = search(r'^EXTRAVERSION =[ \t]*(.*?)$', makefile, MULTILINE)[1]
 full_ver = '.'.join(filter(None, [version, patchlevel, sublevel])) + extra
-data['components']['chromiumos/third_party/kernel']['version'] = full_ver
+data['components']['src/third_party/kernel/v5.4']['version'] = full_ver
 
 # Finally, write the output.
 with open(dirname(__file__) + '/upstream-info.json', 'w') as out:
diff --git a/pkgs/os-specific/linux/chromium-os/upstream-info.json b/pkgs/os-specific/linux/chromium-os/upstream-info.json
index 5264a72a6c9..842912800bc 100644
--- a/pkgs/os-specific/linux/chromium-os/upstream-info.json
+++ b/pkgs/os-specific/linux/chromium-os/upstream-info.json
@@ -1,60 +1,60 @@
 {
   "version": "81.12871.0.0-rc1",
   "components": {
-    "aosp/platform/external/modp_b64": {
-      "name": "modp_b64",
-      "url": "https://chromium.googlesource.com/aosp/platform/external/modp_b64",
-      "rev": "ae26ad8516036774fd3b6e84a98a89955dcf5f61",
-      "sha256": "03ja5fkq3wk0qcnfnqcj09lciky6n6zdj544xs1jfssjhz9yj8hi"
-    },
-    "chromiumos/overlays/chromiumos-overlay": {
-      "name": "chromiumos-overlay",
-      "url": "https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay",
-      "rev": "be285b729ef2af700cc248f741af20c387b5a1bc",
-      "sha256": "18y2icdb3hxqxxaaz8sylw21q2qwcpjyns75ig7ahc4vc1k5hg7s"
-    },
-    "chromiumos/platform/crosvm": {
+    "src/platform/crosvm": {
       "name": "crosvm",
       "url": "https://chromium.googlesource.com/chromiumos/platform/crosvm",
       "rev": "8b8c01e1ad31718932491e4aee63f56109a138e2",
       "sha256": "1qmf1k06pwynh15c3nr9m6v90z2pkk930xniwvlvbvnazrk4rllg"
     },
-    "chromiumos/platform/minigbm": {
+    "src/platform/minigbm": {
       "name": "minigbm",
       "url": "https://chromium.googlesource.com/chromiumos/platform/minigbm",
       "rev": "fbfb338e7db246f05ae68da97141fbd49481584b",
       "sha256": "11p6v4nmqivyqyc8lmypbwqv9q819lhx6v3igfsv3455mmq02kcb"
     },
-    "chromiumos/platform2": {
+    "src/platform2": {
       "name": "platform2",
       "url": "https://chromium.googlesource.com/chromiumos/platform2",
       "rev": "6e0d88d0bea8e7cb28a310298cc8c77ed742f60a",
       "sha256": "13vrbnl3ck8cxiyifc58cs9i0yby92vp5si27ihrx33v6bdlir4s"
     },
-    "chromiumos/third_party/adhd": {
+    "src/third_party/adhd": {
       "name": "adhd",
       "url": "https://chromium.googlesource.com/chromiumos/third_party/adhd",
       "rev": "f361d5b02623274723bff251dafa1e2a2887b013",
       "sha256": "1p8iwjwgmcgmzri03ik2jaid8l0ch0bzn6z9z64dix1hlrvrlliw"
     },
-    "chromiumos/third_party/kernel": {
-      "name": "kernel",
+    "src/third_party/chromiumos-overlay": {
+      "name": "chromiumos-overlay",
+      "url": "https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay",
+      "rev": "be285b729ef2af700cc248f741af20c387b5a1bc",
+      "sha256": "18y2icdb3hxqxxaaz8sylw21q2qwcpjyns75ig7ahc4vc1k5hg7s"
+    },
+    "src/third_party/kernel/v5.4": {
+      "name": "v5.4",
       "url": "https://chromium.googlesource.com/chromiumos/third_party/kernel",
       "rev": "17611b6e636c8ccae25e873e659d69c20b215815",
       "sha256": "0an8w3660v2zf2crb68ambg7vzw2rajpjmcby1am08qy37lqsfkd",
       "version": "5.4.16"
     },
-    "chromiumos/third_party/libqmi": {
+    "src/third_party/libqmi": {
       "name": "libqmi",
       "url": "https://chromium.googlesource.com/chromiumos/third_party/libqmi",
       "rev": "6267c12fef0be9a95d687bf4b476593eeffbaaf1",
       "sha256": "00yjjcngjkbgxmmf79ciddk9yq393a94230ahq8vszyfygzhs3rz"
     },
-    "chromiumos/third_party/modemmanager-next": {
+    "src/third_party/modemmanager-next": {
       "name": "modemmanager-next",
       "url": "https://chromium.googlesource.com/chromiumos/third_party/modemmanager-next",
       "rev": "a4d3a4de1fdab7c282f1a188d2d3d0aaa24f0698",
       "sha256": "1b1l1jfiyl7fnm29pp9npi0gdlqcw1jmmbigf0qlndqdh0ly2izg"
+    },
+    "src/third_party/modp_b64": {
+      "name": "modp_b64",
+      "url": "https://chromium.googlesource.com/aosp/platform/external/modp_b64",
+      "rev": "ae26ad8516036774fd3b6e84a98a89955dcf5f61",
+      "sha256": "03ja5fkq3wk0qcnfnqcj09lciky6n6zdj544xs1jfssjhz9yj8hi"
     }
   }
 }
diff --git a/pkgs/os-specific/linux/kernel/linux-cros.nix b/pkgs/os-specific/linux/kernel/linux-cros.nix
index 6a4a5771b38..789a9fb6fd2 100644
--- a/pkgs/os-specific/linux/kernel/linux-cros.nix
+++ b/pkgs/os-specific/linux/kernel/linux-cros.nix
@@ -3,7 +3,7 @@
 , ... } @ args:
 
 let
-  versionData = upstreamInfo.components."chromiumos/third_party/kernel";
+  versionData = upstreamInfo.components."src/third_party/kernel/v5.4";
 in
 
 with lib;
-- 
2.26.2

^ permalink raw reply	[flat|nested] 16+ messages in thread
end of thread, other threads:[~2020-06-02  8:32 UTC | newest]

Thread overview: 16+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-05-30 19:00 [PATCH 1/2] chromiumOSPackages: improve update.py repo compat Alyssa Ross
2020-05-30 19:00 ` [PATCH 2/2] chromiumOSPackages: 81.12871.0.0-rc1 -> 83.13020.0.0-rc1 Alyssa Ross
2020-05-31  3:01   ` [PATCH v2 " Alyssa Ross
2020-05-31 23:49     ` Cole Helbling
2020-06-01 22:44       ` Alyssa Ross
2020-06-01 23:00         ` Alyssa Ross
2020-06-01 23:38         ` Michael Raskin
2020-06-01 23:55           ` Alyssa Ross
2020-06-02  0:37             ` [PATCH platform2] vm_tools: proto: fix parallel build Alyssa Ross
2020-06-02  0:39               ` Cole Helbling
2020-06-02  0:56   ` [PATCH v3 2/2] chromiumOSPackages: 81.12871.0.0-rc1 -> 83.13020.0.0-rc1 Alyssa Ross
2020-06-02  2:27     ` Cole Helbling
2020-06-02  3:19       ` Alyssa Ross
2020-06-02  8:39       ` [PATCH v3 2/2] chromiumOSPackages: 81.12871.0.0-rc1 ->83.13020.0.0-rc1 Michael Raskin
2020-05-30 23:47 ` [PATCH 1/2] chromiumOSPackages: improve update.py repo compat Cole Helbling
2020-06-02  3:30 ` Alyssa Ross

Code repositories for project(s) associated with this public inbox

	https://spectrum-os.org/git/crosvm
	https://spectrum-os.org/git/doc
	https://spectrum-os.org/git/mktuntap
	https://spectrum-os.org/git/nixpkgs
	https://spectrum-os.org/git/spectrum
	https://spectrum-os.org/git/ucspi-vsock
	https://spectrum-os.org/git/www

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).