- Spectrum Development - spectrum-os.org

[PATCH nixpkgs] spectrumPackages: update execline backtick uses
by Alyssa Ross 12 Aug '21

12 Aug '21

execline 2.8.0.0 changed how backtick's options work: * -i (exit with the program's error code on failure) and -n (chomp off a trailing newline from the captured output) are now the defaults. * -E has been introduced, which does importas automatically. -E is fantastic, because we followed every single use of backtick with importas. Even we didn't have -n before, it's okay that chomping will now be enabled from the default, because in all those cases, there won't be a trailing newline anyway. --- .../linux/spectrum/testhost/default.nix | 14 ++++---------- pkgs/os-specific/linux/spectrum/vm/app/default.nix | 3 +-- pkgs/os-specific/linux/spectrum/vm/net/default.nix | 6 ++---- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pkgs/os-specific/linux/spectrum/testhost/default.nix b/pkgs/os-specific/linux/spectrum/testhost/default.nix index de62f0add67..d81ff8b826e 100644 --- a/pkgs/os-specific/linux/spectrum/testhost/default.nix +++ b/pkgs/os-specific/linux/spectrum/testhost/default.nix @@ -52,18 +52,14 @@ let # routers from clients. Other bits are reserved. # 5-6 Last two octets of client's IP (in 100.64.0.0/16). # - backtick -i router_mac { + backtick -E router_mac { pipeline { printf %.4x ${toString sys-vms.app.vmID} } sed s/^\$..\$\$..\$$/0A:B3:EC:80:\\1:\\2/ } - backtick -i client_mac { + backtick -E client_mac { pipeline { printf %.4x ${toString sys-vms.app.vmID} } sed s/^\$..\$\$..\$$/0A:B3:EC:00:\\1:\\2/ } - multisubstitute { - importas -iu router_mac router_mac - importas -iu client_mac client_mac - } # Create the net VM end, and attach it to the net VM. # @@ -143,12 +139,11 @@ let # is allowed to fail because it might already know that, in # which case it'll return EEXIST. if { modprobe vfio-pci } - backtick -in device_id { + backtick -E device_id { if { dd bs=2 skip=1 count=2 status=none if=''${PCI_PATH}/vendor } if { printf " " } dd bs=2 skip=1 count=2 status=none if=''${PCI_PATH}/device } - importas -iu device_id device_id foreground { redirfd -w 1 /sys/bus/pci/drivers/vfio-pci/new_id printf "%s" $device_id @@ -205,8 +200,7 @@ writeScriptBin "spectrum-testhost" '' if { redirfd -w 1 /proc/sys/net/ipv4/ip_forward echo 1 } importas -iu runtime_dir XDG_RUNTIME_DIR - backtick -in TOP { mktemp -dp $runtime_dir spectrum.XXXXXXXXXX } - importas -iu top TOP + backtick -E top { mktemp -dp $runtime_dir spectrum.XXXXXXXXXX } if { echo $top } if { rsync -r --chmod=Du+w ${servicesDir}/ ''${top}/service } background { diff --git a/pkgs/os-specific/linux/spectrum/vm/app/default.nix b/pkgs/os-specific/linux/spectrum/vm/app/default.nix index f8ff480932c..9ffed7e5b13 100644 --- a/pkgs/os-specific/linux/spectrum/vm/app/default.nix +++ b/pkgs/os-specific/linux/spectrum/vm/app/default.nix @@ -26,12 +26,11 @@ runCommand "vm-app" rec { oneshot ''; up = writeText "net-up" '' - backtick -i LOCAL_IP { + backtick -E LOCAL_IP { pipeline { ip -j link show eth0 } pipeline { jq -r ".[0].address | split(\":\") | .[4:6] | \"0x\" + .[]" } xargs printf "100.64.%d.%d" } - importas -iu LOCAL_IP LOCAL_IP if { ip address add ''${LOCAL_IP}/32 dev eth0 } if { ip link set eth0 up } diff --git a/pkgs/os-specific/linux/spectrum/vm/net/default.nix b/pkgs/os-specific/linux/spectrum/vm/net/default.nix index 5921b62fcf7..75c86b8f92c 100644 --- a/pkgs/os-specific/linux/spectrum/vm/net/default.nix +++ b/pkgs/os-specific/linux/spectrum/vm/net/default.nix @@ -54,12 +54,11 @@ runCommand "vm-net" rec { # Our IP is encoded in the NIC-specific portion of the # interface's MAC address. - backtick -i CLIENT_IP { + backtick -E CLIENT_IP { pipeline { ip -j link show $INTERFACE } pipeline { jq -r ".[0].address | split(\":\") | .[4:6] | \"0x\" + .[]" } xargs printf "100.64.%d.%d" } - importas -iu CLIENT_IP CLIENT_IP if { ip address add 169.254.0.1/32 dev $INTERFACE } if { ip link set $INTERFACE up } @@ -121,7 +120,7 @@ runCommand "vm-net" rec { ''; run = writeScript "connman-run" '' #! ${execline}/bin/execlineb -S0 - backtick -in HARDWARE_INTERFACES { + backtick -E HARDWARE_INTERFACES { pipeline { find -L /sys/class/net -mindepth 2 -maxdepth 2 -name address -print0 } @@ -132,7 +131,6 @@ runCommand "vm-net" rec { # Extract the interface names from the address file paths. awk -F/ "{if (NR > 1) printf \",\"; printf \"%s\", $5}" } - importas -iu HARDWARE_INTERFACES HARDWARE_INTERFACES ${connman}/bin/connmand -ni $HARDWARE_INTERFACES ''; base-commit: d2e147bedf251976ed99b94b6c905d6761f7a892 -- 2.32.0

2 2

This Week in Spectrum, 2021-W31
by Alyssa Ross 08 Aug '21

08 Aug '21

What a week. Progress has felt a bit slow, but the work has been consistently interesting, and there have been some exciting developments in the ecosystem. Nixpkgs ------- Last week I'd posted the patches required to get all of Spectrum working with a more up-to-date Nixpkgs. After last week's email, Cole reviewed the patch set, so I applied it, and merged the nixpkgs-update branch into master, so now Spectrum is using a reasonably up-to-date Nixpkgs for the first time in a long time. I also said > Once that's done, it's time for another chromiumOSPackages upgrade, but > that should be pretty easy this time because we're only one version behind. Alas, it's never that simple. Firstly, the chromiumOSPackages update script broke, because the information about the currently released Chromium OS build seems to have gone missing. Google is apparently[1] serving build number 13982, but their published build metadata[2] includes builds 13981 and 13983, but not 13982. This means it's not possible for me to know what Git revisions are used in the currently released Chromium OS. Assuming this is just a one time thing, I hacked the update script to just look at the previous build, but we should keep an eye on this. If it ever happens again we should probably implement some sort of mitigation in the update script. Once I had new versions of the Chromium OS packages it was time to get them to build, which was straightforward enough for everything except crosvm. For Spectrum, we have a patch[3] for crosvm to make it support VIRTIO_NET_F_MAC[4], which is a mechanism by which the host system can indicate to the guest kernel what it should set as the MAC address of a virtual network device. After the update, this patch no longer applied, because all of a sudden crosvm has two different virtual network device implementations. This turns out to be because crosvm has implemented vhost-user[5], a protocol to allow virtual devices to be implemented outside the VMM program! This is great news, but it's also surprising. virtiofs[6] was designed to be implemented with vhost-user, but when crosvm implemented it a while ago, they became the only implementer to do so in-VMM. It's great to see them moving in the vhost-user direction, because it makes it much easier to mix and match virtual device implementation and VMMs. Most excitingly, I saw a reference to vhost-user-wl, meaning a standalone implementation of Virtio Wayland. This would allow us to use Virtio Wayland with other, non-crosvm VMMs, which is great because I think cloud-hypervisor[7] is probably going to end up being a better fit for Spectrum, but Virtio Wayland was crosvm's killer app. I'd even thought about trying to port crosvm's virtio wayland implementation to vhost-user myself, so it's great to know that when the time comes, that'll already have been done for me. While I could have fixed my crosvm patch for all this new virtual network device code, the introduction of vhost-user support means we should be able to drop the patch altogether. cloud-hypervisor provides a vhost-user-net implementation[8] that already supports VIRTIO_NET_F_MAC, so if we can just get crosvm to talk to cloud-hypervisor's vhost-user-net implementation we shouldn't have to carry any patch for this any more. [1]: https://cros-updates-serving.appspot.com/ [2]: https://chromium.googlesource.com/chromiumos/manifest-versions [3]: https://spectrum-os.org/git/nixpkgs/tree/pkgs/os-specific/linux/chromium-os… [4]: https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html [5]: https://qemu.readthedocs.io/en/latest/interop/vhost-user.html [6]: https://virtio-fs.gitlab.io/index.html#overview [7]: https://github.com/cloud-hypervisor/cloud-hypervisor [8]: https://github.com/cloud-hypervisor/cloud-hypervisor/tree/db2159b5638eb4ccf… cloud-hypervisor / rust-vmm --------------------------- So I started looking at cloud-hypervisor to try to hook this all up, but it looks like cloud-hypervisor still has some issues where it doesn't quite follow the vhost-user specification. As I was trying to debug this, I noticed some UB in rust-vmm[9] (the shared utility code project for crosvm and its derivatives like cloud-hypervisor and Firecracker[10]). Being a good citizen, I started working on a fix for this, and then I encountered some more issues with functions that should have been marked as unsafe but weren't. So that's going to need to be fixed too. But the rust-vmm code is otherwise very high quality and easy to work with, and they're very responsive, so it shouldn't take long to get these issues fixed. The affected code is fortunately all to do with communication between vhost-user backends and the VMM, so it's very very unlikely that it's anything that could be exploited by a guest. These sorts of problems are exactly the sort of thing that Rust is supposed to prevent, so it was disappointing to discover these issues. But on the other hand, Rust made them stand out like sore thumbs to me, so even though these issues managed to sneak in, there's definitely still a huge benefit to using Rust for these sorts of programs. [9]: https://github.com/rust-vmm [10]: https://firecracker-microvm.github.io/ In the next week, I'm hoping to get all the rust-vmm issues I've discovered fixed, and maybe get cloud-hypervisor's backends to speak proper vhost-user. I'm getting my second vaccination next week as well though, so we'll see. I might just end up being sick.

1 0

[COMMIT] nixpkgs-update merged into master
by Alyssa Ross 04 Aug '21

04 Aug '21

Everything seems to be working. I'm going to start another merge immediately, so we're fully up to date. But the Nixpkgs I merged in was only from last week, so it shouldn't be too hard.

1 0

[PATCH nixpkgs 0/6] Fix build failures after Nixpkgs update
by Alyssa Ross 03 Aug '21

03 Aug '21

Alyssa Ross (6): chromiumOSPackages.vm_protos: set full go_package chromiumOSPackages.modemmanager: disable patches chromiumOSPackages.modemmanager-next: add gtk-doc chromiumOSPackages.libqmi: disable QRTR linux_cros: fix build wlroots: update Spectrum patches ...ayland-downgrade-to-wl_compositor-v3.patch | 64 ++++++----- ...til-support-virtio_wl-shm-allocation.patch | 11 +- ...-mk-don-t-leak-source-absolute-paths.patch | 4 +- ...ommon-mk-.gn-don-t-hardcode-env-path.patch | 4 +- .../os-specific/linux/chromium-os/default.nix | 3 +- .../linux/chromium-os/libqmi/default.nix | 4 +- .../chromium-os/modem-manager/default.nix | 2 + .../linux/chromium-os/modem-manager/next.nix | 6 +- ...er-don-t-leak-source-absolute-paths.patch} | 6 +- ...tools-sommelier-Switch-to-the-stabl.patch} | 6 +- .../linux/chromium-os/sommelier/default.nix | 4 +- ...-goproto_library-source_relative-opt.patch | 48 +++++++++ ...tools-proto-set-go_package-correctly.patch | 102 ++++++++++++++++++ .../linux/chromium-os/vm_protos/default.nix | 5 + 14 files changed, 222 insertions(+), 47 deletions(-) rename pkgs/os-specific/linux/chromium-os/sommelier/{0003-sommelier-don-t-leak-source-absolute-paths.patch => 0005-sommelier-don-t-leak-source-absolute-paths.patch} (82%) rename pkgs/os-specific/linux/chromium-os/sommelier/{0004-Revert-Revert-vm_tools-sommelier-Switch-to-the-stabl.patch => 0006-Revert-Revert-vm_tools-sommelier-Switch-to-the-stabl.patch} (99%) create mode 100644 pkgs/os-specific/linux/chromium-os/vm_protos/0003-common-mk-add-goproto_library-source_relative-opt.patch create mode 100644 pkgs/os-specific/linux/chromium-os/vm_protos/0004-vm_tools-proto-set-go_package-correctly.patch base-commit: cab13e087c80c4e7476bb7df432e2b7deb56563d -- 2.32.0

2 8

This (Last) Week in Spectrum, 2021-W30
by Alyssa Ross 02 Aug '21

02 Aug '21

Hi, it's been a long time since I've done one of these, but so much interesting stuff happened this week, and there's been a bit of renewed interest in the project, that it _really_ needed an update. So here it is. As always when I'm not keeping up with TWiS, this update is limited to only things that happened in the last week, because otherwise it would just be far too long. Wayland ------- A problem I've been thinking about for a long time is Wayland access control. The Linux desktop ecosystem is moving towards access controls for most functionality through xdg-desktop-portal[1] (which sadly doesn't seem to have a website). But it doesn't cover stuff that's part of the core Wayland protocol, like access to the clipboard. And some compositors (wlroots-based ones) provide extra Wayland protocols for things like screenshots. And up to now, Wayland compositors haven't really done any authorization for these potentially dangerous APIs. (All of this is only really meaningful for sandboxed applications -- running in Flatpak or a Spectrum VM or something. Linux isn't really set up to do separation between processes running as the same user without namespaces.) So earlier this week, I posted[2] to the Wayland mailing list with an idea I'd been thinking about for a while[3], which was to place a proxy program in front of the Wayland compositor, that would intercept client->compositor requests and handle access control. I was quickly convinced that a proxy wasn't a good idea, but there was a lot of discussion, and it was really helpful to me figuring out what the right way to do it might be. There are really two problems to be solved here, one of which I hadn't even thought much about. The first is securely identifying a Wayland client. A compositor needs to be able to form the question "Should client X be able to do this?", and to do that it needs to be able to identify a client as client X, and know if it tries to interact with client X later, that it'll be talking to the same client. In a non-virtualized system, the obvious way to do this would be getting the pid of the client from the connection socket and then looking it up in proc(5) to find out executable path, but this approach is fundamentally insecure[4]. The way forward here (and one that would work for Spectrum) appears to be the proposed Security Contexts protocol[5], which would allow a sandbox implementation to provide a security context identifier for a client before handing off the Wayland connection to the client. Once the security context had been set up, it wouldn't be allowed to be modified, so once the sandboxed client was given the connection, it wouldn't be able to change the security context identifier. In Spectrum, the security context identifier here would likely be a unique, user-provided name for the VM, and the security context setup would be done by the virtio wayland implementation in the VMM. The second part of this puzzle is how the compositor should decide whether a client should be allowed to perform a particular operation, like a paste or going fullscreen (which is risky because it might spoof a desktop). There was actually a previous attempt at this a long time ago. libwsm[7] (short for Wayland Security Module) was a library that compositors would have integrated to make authorization decisions. But it wasn't adopted by compositors, and it some things we now know[8] to be bad ideas, like setting policy based on the executable path. It also made compositors responsible for any sort of authorization UI. In my opinion, it's better to have that done by the external piece, so that compositors have as little work to do as possible and therefore authorization is implemented as widely as possible. The compositor could implement an authorization system entirely on its own, but this would be a lot of code for each compositor to write, and it would limit the user to whatever permissions system the compositor came up with, which might not be able to accommodate their needs. (An example of this would be a Spectrum user that wanted to allow pasting between two applications, but not allow pasting between either of those and any other application.) It could also be implemented by having the compositors integrate with something like polkit[6], but compositor authors are reluctant to integrate directly with a single system like that, and even polkit might not support everything that would be desirable in an authorization system. (For example, it might be nice to implement libwsm's concept of a soft allow, where an action is permitted, but a notification is shown so the user is aware it's happened.) So a third solution, suggested by ifreund, a Wayland compositor author, in the Spectrum IRC channel, would be to have a privileged Wayland client that the compositor could ask authorization questions to, with a new protocol. The compositor would know that this particular client should be privileged because it either wouldn't be in a security context, or would be given a special security context identifier known to the compositor ahead of time. Then, implementations of this protocol could do authorization however they wanted, with the only limitation being the questions the compositor was programmed to ask them. I think this is a good way forward, but it'd be important to discuss with more compositor authors before getting to excited about it. Next steps from here with Wayland are: * Figure out what needs to happen to move the security contexts proposal forward. If it needs an experimental implementation, maybe we could help with that? * Inquire about the authorization protocol idea, and see how other compositor offers would feel about it. If there's a generally positive reaction, figure out how to move forward with it. [1]: https://github.com/flatpak/xdg-desktop-portal/ [2]: https://lists.freedesktop.org/archives/wayland-devel/2021-July/041896.html [3]: https://spectrum-os.org/lists/archives/spectrum-discuss/8735ueudel.fsf@alys… [4]: https://gitlab.freedesktop.org/wayland/weston/-/issues/206#note_176699 [5]: https://gitlab.freedesktop.org/wayland/wayland-protocols/-/merge_requests/68 [6]: https://www.freedesktop.org/software/polkit/docs/latest/polkit.8.html [7]: https://github.com/mupuf/libwsm [8]: https://lore.kernel.org/lkml/87r1xplsku.fsf@x220.int.ebiederm.org/ Nixpkgs ------- I created a branch in Spectrum's nixpkgs repository[9] for the long-overdue merge with upstream, did the merge, and posted some patches[10] to spectrum-devel that fix all the builds that broke as a result. I've been letting them sit for a few days hoping for a review. Once that's done, it's time for another chromiumOSPackages upgrade, but that should be pretty easy this time because we're only one version behind. [9]: https://spectrum-os.org/git/nixpkgs [10]: https://spectrum-os.org/lists/archives/spectrum-devel/20210729100928.196534… Thanks for keeping up with Spectrum. :)

1 0

Updating Spectrum's nixpkgs
by Alyssa Ross 29 Jul '21

29 Jul '21

I've just pushed a branch to our nixpkgs repository called nixpkgs-update, where I've merged in yesterday's nixos-unstable. From here, I'm going to send some patches targeting that branch, until everything builds again and we can merge it back into master. And then we're never going to go this long between updates ever again.

1 0

[PATCH nixpkgs 0/2] spectrumPackages.spectrum-vm: switch compositor from Wayfire to tinywl
by Alyssa Ross 01 Jul '21

01 Jul '21

This is another change I want to get in before updating Spectrum's Nixpkgs, because once we do that the current approach will no longer work. As explained in patch 2's commit message, this change only affects this demo, not the eventual Spectrum system. Alyssa Ross (2): tinywl: init at 0.11.0 spectrumPackages.sys-vms.comp: switch to tinywl .../window-managers/tinywl/default.nix | 30 ++++++++++++++++++ .../linux/spectrum/vm/comp/default.nix | 31 ++++++------------- pkgs/top-level/all-packages.nix | 2 ++ 3 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 pkgs/applications/window-managers/tinywl/default.nix base-commit: 129d329e3bcdcc972454ce284e17da16b28bd401 -- 2.32.0

1 2

[PATCH nixpkgs 0/2] Pre-merge fixes for Nixpkgs update
by Alyssa Ross 01 Jul '21

01 Jul '21

I'm working on merging in a new version of upstream Nixpkgs. These are a couple of small fixes that came up during that work, but can be taken care of first since they make improvements to the current version. Alyssa Ross (2): chromiumOSPackages.modemmanager-next: fix libqmi spectrumPackages.sys-vms.comp: don't pin Emacs .../linux/chromium-os/modem-manager/next.nix | 12 ++++++++++-- pkgs/os-specific/linux/spectrum/vm/comp/default.nix | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) base-commit: 129d329e3bcdcc972454ce284e17da16b28bd401 -- 2.31.1

2 5

[PATCH platform2 0/2] Make vm_protos compatible with protoc-gen-go 1.5.x
by Alyssa Ross 30 Jun '21

30 Jun '21

My previous attempt[1] at this was totally wrong, because I didn't understand what change needed to be made, and didn't have any way of testing it. To make sure this time that I got it right, I wrote a package for tremplin[2], which was the only Chromium OS component I could find that was written in Go and actually used vm_protos. With these changes, I was able to add "${vm_protos}/lib/gopath" to tremplin's extraSrcPaths, and get a successful build. [1]: https://spectrum-os.org/lists/archives/spectrum-devel/20210627165035.899276… [2]: https://chromium.googlesource.com/chromiumos/platform/tremplin Alyssa Ross (2): common-mk: add goproto_library source_relative opt vm_tools: proto: set go_package correctly common-mk/proto_library.gni | 7 +++++++ vm_tools/proto/BUILD.gn | 5 +++++ vm_tools/proto/tremplin.proto | 2 +- vm_tools/proto/vm_crash.proto | 2 +- vm_tools/proto/vm_guest.proto | 1 + vm_tools/proto/vm_host.proto | 1 + 6 files changed, 16 insertions(+), 2 deletions(-) -- 2.31.1

2 4

[PATCH platform2] vm_tools: proto: fix go_package import path
by Alyssa Ross 27 Jun '21

27 Jun '21

This is required by protoc-gen-go >= 1.4.0. --- I don't really have any idea what I'm doing here -- this is my best guess. If somebody could tell me if it looks sensible I'd be much obliged. vm_tools/proto/common.proto | 1 + vm_tools/proto/tremplin.proto | 2 +- vm_tools/proto/vm_crash.proto | 2 +- vm_tools/proto/vm_guest.proto | 1 + vm_tools/proto/vm_host.proto | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vm_tools/proto/common.proto b/vm_tools/proto/common.proto index 39a3e39528..f11d128ef1 100644 --- a/vm_tools/proto/common.proto +++ b/vm_tools/proto/common.proto @@ -8,6 +8,7 @@ option cc_enable_arenas = true; // Common definitions used by both host and guest messages. package vm_tools; +option go_package = "chromiumos/vm_tools"; // gRPC requires that every RPC has an argument and a return value. This empty // message is used when an argument or return value is not required. diff --git a/vm_tools/proto/tremplin.proto b/vm_tools/proto/tremplin.proto index aac76f7a9e..0a8968fe8d 100644 --- a/vm_tools/proto/tremplin.proto +++ b/vm_tools/proto/tremplin.proto @@ -8,7 +8,7 @@ option cc_enable_arenas = true; // This file defines services for tremplin, the container springboard service. package vm_tools.tremplin; -option go_package = "tremplin_proto"; +option go_package = "chromiumos/vm_tools/tremplin/tremplin_proto"; // This needs to be duplicated because the gyp rule for building // go code makes it difficult to have imports. diff --git a/vm_tools/proto/vm_crash.proto b/vm_tools/proto/vm_crash.proto index 6e4f62fe13..70d620fde8 100644 --- a/vm_tools/proto/vm_crash.proto +++ b/vm_tools/proto/vm_crash.proto @@ -7,7 +7,7 @@ syntax = "proto3"; option cc_enable_arenas = true; package vm_tools.cicerone; -option go_package = "vm_crash"; +option go_package = "chromiumos/vm_tools/cicerone/vm_crash"; import "common.proto"; diff --git a/vm_tools/proto/vm_guest.proto b/vm_tools/proto/vm_guest.proto index 34d2fd3d60..b3a742ec07 100644 --- a/vm_tools/proto/vm_guest.proto +++ b/vm_tools/proto/vm_guest.proto @@ -8,6 +8,7 @@ option cc_enable_arenas = true; // This file defines services that will be running in the guest VM. package vm_tools; +option go_package = "chromiumos/vm_tools"; import "common.proto"; import "google/protobuf/timestamp.proto"; diff --git a/vm_tools/proto/vm_host.proto b/vm_tools/proto/vm_host.proto index a8bd066f61..de31fa747f 100644 --- a/vm_tools/proto/vm_host.proto +++ b/vm_tools/proto/vm_host.proto @@ -8,6 +8,7 @@ option cc_enable_arenas = true; // This file defines services that will be running on the host for the VM. package vm_tools; +option go_package = "chromiumos/vm_tools/vm_host"; import "common.proto"; -- 2.31.1

1 0