host: allow VMs to be powered off

Before this change, the s6 services for cloud-hypervisor and virtiofsd
were only started when a VM was started, and vm-stop would bring the
service down.  The problem with this was that if a VM powered itself
off, instead of being stopped on the host using vm-stop, the VM would
instantly be restarted by s6.

To fix this, we disentangle keeping cloud-hypervisor running from
keeping the VM running.  cloud-hypervisor will now always be running,
so s6 will never restart it in normal operation, but it won't be
running a VM until it's told to.  Accomplishing this means having
start-vmm (renamed from start-vm to reflect its new purpose) configure
the VM in cloud-hypervisor without booting it, which is only possible
using the API, not the command line.  As a result, start-vm now
depends on miniserde so that it can construct the VM config JSON
object required by the API.

The build of start-vm has been adjusted to accomodate the complexity
stemming from the new dependencies.  Tests are moved into passthru,
because the start-vm used in Spectrum should have panic=abort, but
tests need panic=unwind, and we can't use both in the same Meson
instance without duplicating the non-native dependencies.

We can't use s6-rc dependencies to automatically boot provider VMs in
this setup, so vm-start has been modified to recurse into provider VMs.

lsvm has been updated to check the Cloud Hypervisor API to see whether
a VM is running, rather than just checking to see whether the s6
service is up.

Because cloud-hypervisor is now to be started as early as possible, we
need to make the dependencies of ext-rc-init more precise, so that
cloud-hypervisor does not attempt to start before /dev/kvm or
/dev/net/tun is available.

We're not using Meson's support for Cargo subprojects yet, because it
currently always builds crates with all features enabled.

Signed-off-by: Alyssa Ross <hi@alyssa.is>

74 files changed, 1157 insertions, 625 deletions

diff --git a/.codespellrc b/.codespellrc
index 7240720..385d25f 100644
--- a/.codespellrc
+++ b/.codespellrc
@@ -2,4 +2,4 @@
 # SPDX-License-Identifier: CC0-1.0
 
 [codespell]
-ignore-words-list = crate,rouge
+ignore-words-list = crate,rouge,ser
diff --git a/.gitignore b/.gitignore
index e9858b9..0491ebb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,11 @@
 # SPDX-License-Identifier: CC0-1.0
-# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is>
+# SPDX-FileCopyrightText: 2021-2023 Alyssa Ross <hi@alyssa.is>
 
 /config.nix
 build/
 result
 result-*
+
+**/subprojects/*
+!**/subprojects/*.wrap
+!**/subprojects/packagefiles
diff --git a/host/rootfs/Makefile b/host/rootfs/Makefile
index a108a29..fc61c09 100644
--- a/host/rootfs/Makefile
+++ b/host/rootfs/Makefile
@@ -78,6 +78,8 @@ S6_RC_FILES = \
 	etc/s6-rc/ext-rc/type \
 	etc/s6-rc/ext/type \
 	etc/s6-rc/ext/up \
+	etc/s6-rc/kvm/type \
+	etc/s6-rc/kvm/up \
 	etc/s6-rc/mdevd-coldplug/dependencies \
 	etc/s6-rc/mdevd-coldplug/type \
 	etc/s6-rc/mdevd-coldplug/up \
diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix
index 5bd2488..c6664bd 100644
--- a/host/rootfs/default.nix
+++ b/host/rootfs/default.nix
@@ -6,7 +6,7 @@ import ../../lib/call-package.nix (
 { callSpectrumPackage, lseek, src, pkgsMusl, pkgsStatic, linux_latest }:
 pkgsStatic.callPackage (
 
-{ start-vm
+{ start-vmm
 , lib, stdenvNoCC, nixos, runCommand, writeReferencesToFile, erofs-utils, s6-rc
 , busybox, cloud-hypervisor, cryptsetup, execline, e2fsprogs, jq, kmod
 , mdevd, s6, s6-linux-init, socat, util-linuxMinimal, virtiofsd, xorg
@@ -44,7 +44,7 @@ let
 
   packages = [
     cloud-hypervisor e2fsprogs execline jq kmod mdevd
-    s6 s6-linux-init s6-rc socat start-vm virtiofsd
+    s6 s6-linux-init s6-rc socat start-vmm virtiofsd
 
     (cryptsetup.override {
       programs = {
diff --git a/host/rootfs/etc/mdev.conf b/host/rootfs/etc/mdev.conf
index 1d6b630..31af4d1 100644
--- a/host/rootfs/etc/mdev.conf
+++ b/host/rootfs/etc/mdev.conf
@@ -3,5 +3,5 @@
 
 -$MODALIAS=.* 0:0 660 +/etc/mdev/modalias.sh
 -$DEVTYPE=(disk|partition) 0:0 660 +/etc/mdev/block/add
-kvm 0:0 660
+kvm 0:0 660 +background { /etc/mdev/listen kvm }
 dri/card0 0:0 660 +background { /etc/mdev/listen card0 }
diff --git a/host/rootfs/etc/s6-rc/ext-rc-init/dependencies b/host/rootfs/etc/s6-rc/ext-rc-init/dependencies
index ba3cd47..c8fb026 100644
--- a/host/rootfs/etc/s6-rc/ext-rc-init/dependencies
+++ b/host/rootfs/etc/s6-rc/ext-rc-init/dependencies
@@ -3,3 +3,5 @@
 #
 core
 ext
+kvm
+static-nodes
diff --git a/host/rootfs/etc/s6-rc/ext-rc-init/up b/host/rootfs/etc/s6-rc/ext-rc-init/up
index b2c2595..b75085d 100644
--- a/host/rootfs/etc/s6-rc/ext-rc-init/up
+++ b/host/rootfs/etc/s6-rc/ext-rc-init/up
@@ -2,7 +2,8 @@
 # SPDX-FileCopyrightText: 2021-2023 Alyssa Ross <hi@alyssa.is>
 # SPDX-FileCopyrightText: 2022 Unikie
 
-if { mkdir -p /run/s6-rc.ext.src }
+if { mkdir -p /run/s6-rc.ext.src/ok-vmm/contents.d }
+if { redirfd -w 1 /run/s6-rc.ext.src/ok-vmm/type echo bundle }
 cd /run/s6-rc.ext.src
 
 if {
@@ -16,8 +17,9 @@ if {
   if { mkdir vm-${name} vm-${name}/dependencies.d vm-${name}/env }
   if { redirfd -w 1 vm-${name}/type echo longrun }
   if { redirfd -w 1 vm-${name}/notification-fd echo 3 }
-  if { redirfd -w 1 vm-${name}/run printf "#!/bin/execlineb -P\n/bin/start-vm" }
+  if { redirfd -w 1 vm-${name}/run printf "#!/bin/execlineb -P\n/bin/start-vmm" }
   if { chmod +x vm-${name}/run }
+  if { touch ok-vmm/contents.d/vm-${name} }
 
   if {
     elglob -0 paths ${dir}/shared-dirs/*
@@ -43,4 +45,5 @@ if {
 }
 
 if { s6-rc-compile /run/s6-rc.ext.db /run/s6-rc.ext.src }
-s6-rc-init -c /run/s6-rc.ext.db -l /run/s6-rc.ext -p ext- /run/service
+if { s6-rc-init -c /run/s6-rc.ext.db -l /run/s6-rc.ext -p ext- /run/service }
+s6-rc -ul /run/s6-rc.ext change ok-vmm
diff --git a/host/rootfs/etc/s6-rc/kvm/type b/host/rootfs/etc/s6-rc/kvm/type
new file mode 100644
index 0000000..bdd22a1
--- /dev/null
+++ b/host/rootfs/etc/s6-rc/kvm/type
@@ -0,0 +1 @@
+oneshot
diff --git a/host/rootfs/etc/s6-rc/kvm/type.license b/host/rootfs/etc/s6-rc/kvm/type.license
new file mode 100644
index 0000000..a941ca4
--- /dev/null
+++ b/host/rootfs/etc/s6-rc/kvm/type.license
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: CC0-1.0
+SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
diff --git a/host/rootfs/etc/s6-rc/kvm/up b/host/rootfs/etc/s6-rc/kvm/up
new file mode 100644
index 0000000..c02e3f9
--- /dev/null
+++ b/host/rootfs/etc/s6-rc/kvm/up
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: EUPL-1.2+
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+/etc/mdev/wait kvm
diff --git a/host/rootfs/usr/bin/lsvm b/host/rootfs/usr/bin/lsvm
index b5a979e..a2b2b2b 100755
--- a/host/rootfs/usr/bin/lsvm
+++ b/host/rootfs/usr/bin/lsvm
@@ -1,17 +1,21 @@
 #!/bin/execlineb -P
 # SPDX-License-Identifier: EUPL-1.2+
-# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
 
-foreground { s6-rc -bu change ext }
 foreground { printf "NAME                \tSTATUS\n" }
 cd /ext/svc/data
 elglob -0 vms *
 forx -E vm { $vms }
 if { printf "%-20s\t" $vm }
-ifte {
-  ifte { echo "STOPPED" }
-  { echo "RUNNING" }
-  test -f /run/service/ext-vm-${vm}/down
+if -n {
+  redirfd -w 2 /dev/null
+  backtick -E state {
+    pipeline -w { jq -r .state }
+    ch-remote --api-socket /run/service/ext-vm-${vm}/env/cloud-hypervisor.sock info
+  }
+  case -s $state {
+    Created { echo "STOPPED" }
+  }
+  echo "RUNNING"
 }
-{ echo "UNKNOWN" }
-test -d /run/service/ext-vm-${vm}
+echo "UNKNOWN"
diff --git a/host/rootfs/usr/bin/vm-start b/host/rootfs/usr/bin/vm-start
index 46668eb..effc65d 100755
--- a/host/rootfs/usr/bin/vm-start
+++ b/host/rootfs/usr/bin/vm-start
@@ -1,6 +1,15 @@
 #!/bin/execlineb -S1
 # SPDX-License-Identifier: EUPL-1.2+
-# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
 
 foreground { s6-rc -bu change ext-rc }
-s6-rc -l /run/s6-rc.ext -u change vm-${1}
+
+foreground {
+  redirfd -w 2 /dev/null
+  cd /ext/svc/data/${1}/providers/net
+  elglob -0 providers *
+  forx -pE provider { $providers }
+  vm-start $provider
+}
+
+ch-remote --api-socket /run/service/ext-vm-${1}/env/cloud-hypervisor.sock boot
diff --git a/host/rootfs/usr/bin/vm-stop b/host/rootfs/usr/bin/vm-stop
index 2322003..db2e9f3 100755
--- a/host/rootfs/usr/bin/vm-stop
+++ b/host/rootfs/usr/bin/vm-stop
@@ -1,5 +1,5 @@
 #!/bin/execlineb -S1
 # SPDX-License-Identifier: EUPL-1.2+
-# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
 
-s6-rc -l /run/s6-rc.ext -d change vm-${1}
+ch-remote --api-socket /run/service/ext-vm-${1}/env/cloud-hypervisor.sock shutdown
diff --git a/host/start-vm/ch.rs b/host/start-vm/ch.rs
deleted file mode 100644
index 876a6ed..0000000
--- a/host/start-vm/ch.rs
+++ /dev/null
@@ -1,126 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-use std::ffi::{CStr, OsStr, OsString};
-use std::num::NonZeroI32;
-use std::os::raw::{c_char, c_int};
-use std::os::unix::prelude::*;
-use std::process::{Command, Stdio};
-
-use crate::format_mac;
-
-// Trivially safe.
-const EPERM: NonZeroI32 = unsafe { NonZeroI32::new_unchecked(1) };
-const EPROTO: NonZeroI32 = unsafe { NonZeroI32::new_unchecked(71) };
-
-fn command(vm_name: &OsStr, s: impl AsRef<OsStr>) -> Command {
-    let mut api_socket_path = OsString::from("/run/service/ext-vm-");
-    api_socket_path.push(vm_name);
-    api_socket_path.push("/env/cloud-hypervisor.sock");
-
-    let mut command = Command::new("ch-remote");
-    command.stdin(Stdio::null());
-    command.arg("--api-socket");
-    command.arg(api_socket_path);
-    command.arg(s);
-    command
-}
-
-pub fn add_net(vm_name: &OsStr, tap: RawFd, mac: &str) -> Result<OsString, NonZeroI32> {
-    let mut ch_remote = command(vm_name, "add-net")
-        .arg(format!("fd={},mac={}", tap, mac))
-        .stdout(Stdio::piped())
-        .spawn()
-        .or(Err(EPERM))?;
-
-    let jq_out = match Command::new("jq")
-        .args(["-j", ".id"])
-        .stdin(ch_remote.stdout.take().unwrap())
-        .stderr(Stdio::inherit())
-        .output()
-    {
-        Ok(o) => o,
-        Err(_) => {
-            // Try not to leave a zombie.
-            let _ = ch_remote.kill();
-            let _ = ch_remote.wait();
-            return Err(EPERM);
-        }
-    };
-
-    if let Ok(ch_remote_status) = ch_remote.wait() {
-        if ch_remote_status.success() && jq_out.status.success() {
-            return Ok(OsString::from_vec(jq_out.stdout));
-        }
-    }
-
-    Err(EPROTO)
-}
-
-pub fn remove_device(vm_name: &OsStr, device_id: &OsStr) -> Result<(), NonZeroI32> {
-    let ch_remote = command(vm_name, "remove-device")
-        .arg(device_id)
-        .status()
-        .or(Err(EPERM))?;
-
-    if ch_remote.success() {
-        Ok(())
-    } else {
-        Err(EPROTO)
-    }
-}
-
-/// # Safety
-///
-/// - `vm_name` must point to a valid C string.
-/// - `tap` must be a file descriptor describing an tap device.
-/// - `mac` must be a valid pointer.
-#[export_name = "ch_add_net"]
-unsafe extern "C" fn add_net_c(
-    vm_name: *const c_char,
-    tap: RawFd,
-    mac: *const [u8; 6],
-    id: *mut *mut OsString,
-) -> c_int {
-    let vm_name = CStr::from_ptr(vm_name);
-    let mac = format_mac(&*mac);
-
-    match add_net(OsStr::from_bytes(vm_name.to_bytes()), tap, &mac) {
-        Err(e) => e.get(),
-        Ok(id_str) => {
-            if !id.is_null() {
-                let token = Box::into_raw(Box::new(id_str));
-                *id = token;
-            }
-            0
-        }
-    }
-}
-
-/// # Safety
-///
-/// - `vm_name` must point to a valid C string.
-/// - `id` must be a device ID obtained by calling `add_net_c`.  After
-///   calling `remove_device_c`, the pointer is no longer valid.
-#[export_name = "ch_remove_device"]
-unsafe extern "C" fn remove_device_c(vm_name: *const c_char, device_id: *mut OsString) -> c_int {
-    let vm_name = CStr::from_ptr(vm_name);
-    let device_id = Box::from_raw(device_id);
-
-    if let Err(e) = remove_device(OsStr::from_bytes(vm_name.to_bytes()), device_id.as_ref()) {
-        e.get()
-    } else {
-        0
-    }
-}
-
-/// # Safety
-///
-/// `id` must be a device ID obtained by calling `add_net_c`.  After
-/// calling `device_free`, the pointer is no longer valid.
-#[export_name = "ch_device_free"]
-unsafe extern "C" fn device_free(id: *mut OsString) {
-    if !id.is_null() {
-        drop(Box::from_raw(id))
-    }
-}
diff --git a/host/start-vm/default.nix b/host/start-vm/default.nix
deleted file mode 100644
index 1777d8a..0000000
--- a/host/start-vm/default.nix
+++ /dev/null
@@ -1,39 +0,0 @@
-# SPDX-License-Identifier: MIT
-# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
-
-import ../../lib/call-package.nix (
-{ src, lib, stdenv, meson, ninja, rustc, clippy, run-spectrum-vm }:
-
-stdenv.mkDerivation (finalAttrs: {
-  name = "start-vm";
-
-  src = lib.fileset.toSource {
-    root = ../..;
-    fileset = lib.fileset.intersection src ./.;
-  };
-  sourceRoot = "source/host/start-vm";
-
-  nativeBuildInputs = [ meson ninja rustc ];
-
-  mesonFlags = [ "-Dwerror=true" ];
-
-  doCheck = true;
-
-  passthru.tests = {
-    clippy = finalAttrs.finalPackage.overrideAttrs (
-      { nativeBuildInputs ? [], ... }:
-      {
-        nativeBuildInputs = nativeBuildInputs ++ [ clippy ];
-        RUSTC = "clippy-driver";
-        postBuild = ''touch $out && exit 0'';
-      }
-    );
-
-    run = run-spectrum-vm.override { start-vm = finalAttrs.finalPackage; };
-  };
-
-  meta = {
-    mainProgram = "start-vm";
-  };
-})
-) (_: {})
diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs
deleted file mode 100644
index 0c1e0d2..0000000
--- a/host/start-vm/lib.rs
+++ /dev/null
@@ -1,197 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
-// SPDX-FileCopyrightText: 2022 Unikie
-
-mod ch;
-mod net;
-mod s6;
-mod unix;
-
-use std::borrow::Cow;
-use std::env::args_os;
-use std::ffi::{CString, OsStr, OsString};
-use std::fs::remove_file;
-use std::io::{self, ErrorKind};
-use std::os::unix::net::UnixListener;
-use std::os::unix::prelude::*;
-use std::path::Path;
-use std::process::Command;
-
-use net::{format_mac, net_setup, NetConfig};
-use unix::clear_cloexec;
-
-pub use s6::notify_readiness;
-
-pub fn prog_name() -> String {
-    args_os()
-        .next()
-        .as_ref()
-        .map(Path::new)
-        .and_then(Path::file_name)
-        .map(OsStr::to_string_lossy)
-        .unwrap_or(Cow::Borrowed("start-vm"))
-        .into_owned()
-}
-
-pub fn create_api_socket() -> Result<UnixListener, String> {
-    let _ = remove_file("env/cloud-hypervisor.sock");
-    let api_socket = UnixListener::bind("env/cloud-hypervisor.sock")
-        .map_err(|e| format!("creating API socket: {e}"))?;
-
-    // Safe because we own api_socket.
-    if unsafe { clear_cloexec(api_socket.as_fd()) } == -1 {
-        let errno = io::Error::last_os_error();
-        return Err(format!("clearing CLOEXEC on API socket fd: {}", errno));
-    }
-
-    Ok(api_socket)
-}
-
-pub fn vm_command(
-    service_dir: &Path,
-    vm_dir: &Path,
-    api_socket_fd: RawFd,
-) -> Result<Command, String> {
-    let vm_name = service_dir
-        .file_name()
-        .ok_or_else(|| "directory has no name".to_string())?
-        .as_bytes();
-
-    if !vm_name.starts_with(b"vm-") {
-        return Err("not running from a VM service directory".to_string());
-    }
-
-    if vm_name.contains(&b',') {
-        return Err(format!("VM name may not contain a comma: {:?}", vm_name));
-    }
-
-    let vm_name = OsStr::from_bytes(&vm_name[3..]);
-
-    let config_dir = vm_dir.join(vm_name).join("config");
-
-    let mut command = Command::new("cloud-hypervisor");
-    command.args(["--api-socket", &format!("fd={api_socket_fd}")]);
-    command.args(["--cmdline", "console=ttyS0 root=PARTLABEL=root"]);
-    command.args(["--memory", "size=256M,shared=on"]);
-    command.args(["--console", "pty"]);
-    command.arg("--kernel");
-    command.arg(config_dir.join("vmlinux"));
-
-    let net_providers_dir = config_dir.join("providers/net");
-    match net_providers_dir.read_dir() {
-        Ok(entries) => {
-            // TODO: to support multiple net providers, we'll need
-            // a better naming scheme for tap and bridge devices.
-            #[allow(clippy::never_loop)]
-            for r in entries {
-                let entry = r
-                    .map_err(|e| format!("examining directory entry: {}", e))?
-                    .file_name();
-
-                // Safe because provider_name is the name of a directory entry, so
-                // can't contain a null byte.
-                let provider_name = unsafe { CString::from_vec_unchecked(entry.into_vec()) };
-
-                // Safe because we pass a valid pointer and check the result.
-                let NetConfig { fd, mac } = unsafe { net_setup(provider_name.as_ptr()) };
-                if fd == -1 {
-                    let e = io::Error::last_os_error();
-                    return Err(format!("setting up networking failed: {}", e));
-                }
-
-                command
-                    .arg("--net")
-                    .arg(format!("fd={},mac={}", fd, format_mac(&mac)));
-
-                break;
-            }
-        }
-        Err(e) if e.kind() == ErrorKind::NotFound => {}
-        Err(e) => return Err(format!("reading directory {:?}: {}", net_providers_dir, e)),
-    }
-
-    let blk_dir = config_dir.join("blk");
-    match blk_dir.read_dir().map(Iterator::peekable) {
-        Ok(mut entries) => {
-            if entries.peek().is_some() {
-                command.arg("--disk");
-            }
-
-            for result in entries {
-                let entry = result
-                    .map_err(|e| format!("examining directory entry: {}", e))?
-                    .path();
-
-                if entry.extension() != Some(OsStr::new("img")) {
-                    continue;
-                }
-
-                if entry.as_os_str().as_bytes().contains(&b',') {
-                    return Err(format!("illegal ',' character in path {:?}", entry));
-                }
-
-                let mut arg = OsString::from("path=");
-                arg.push(entry);
-                arg.push(",readonly=on");
-                command.arg(arg);
-            }
-        }
-        Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)),
-    }
-
-    if config_dir.join("wayland").exists() {
-        command.arg("--gpu").arg({
-            let mut gpu = OsString::from("socket=../gpu-");
-            gpu.push(vm_name);
-            gpu.push("/env/crosvm.sock");
-            gpu
-        });
-    }
-
-    let shared_dirs_dir = config_dir.join("shared-dirs");
-    match shared_dirs_dir.read_dir().map(Iterator::peekable) {
-        Ok(mut entries) => {
-            if entries.peek().is_some() {
-                command.arg("--fs");
-            }
-
-            for result in entries {
-                let entry = result
-                    .map_err(|e| format!("examining directory entry: {}", e))?
-                    .file_name();
-
-                let mut arg = OsString::from("tag=");
-                arg.push(&entry);
-                arg.push(",socket=../fs-");
-                arg.push(vm_name);
-                arg.push("-");
-                arg.push(&entry);
-                arg.push("/env/virtiofsd.sock");
-                command.arg(arg);
-            }
-        }
-        Err(e) if e.kind() == ErrorKind::NotFound => {}
-        Err(e) => return Err(format!("reading directory {:?}: {}", shared_dirs_dir, e)),
-    }
-
-    command.arg("--serial").arg({
-        let mut serial = OsString::from("file=/run/");
-        serial.push(vm_name);
-        serial.push(".log");
-        serial
-    });
-
-    Ok(command)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_vm_name_comma() {
-        assert!(vm_command(Path::new("/vm-,"), Path::new(""), -1)
-            .unwrap_err()
-            .contains("comma"));
-    }
-}
diff --git a/host/start-vm/meson.build b/host/start-vm/meson.build
deleted file mode 100644
index d059e3b..0000000
--- a/host/start-vm/meson.build
+++ /dev/null
@@ -1,20 +0,0 @@
-# SPDX-License-Identifier: EUPL-1.2+
-# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-project('start-vm', 'rust', 'c',
-  default_options : ['c_std=c2x', 'rust_std=2018', 'warning_level=3'])
-
-add_project_arguments('-D_GNU_SOURCE', '-Wno-error=attributes', language : 'c')
-add_project_arguments('-C', 'panic=abort', language : 'rust')
-
-c_lib = static_library('start-vm', 'net.c', 'net-util.c', 'unix.c')
-rust_lib = static_library('start_vm', 'lib.rs', link_with : c_lib)
-
-executable('start-vm', 'start-vm.rs', link_with : rust_lib, install : true)
-
-test_exe = executable('start-vm-test', 'lib.rs',
-  rust_args : ['--test', '-C', 'panic=unwind'],
-  link_with : c_lib)
-test('Rust unit tests', test_exe, protocol : 'rust')
-
-subdir('tests')
diff --git a/host/start-vm/net.rs b/host/start-vm/net.rs
deleted file mode 100644
index 7c73fa0..0000000
--- a/host/start-vm/net.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-use std::os::raw::c_char;
-
-#[repr(C)]
-pub struct NetConfig {
-    pub fd: i32,
-    pub mac: [u8; 6],
-}
-
-extern "C" {
-    pub fn net_setup(provider_vm_name: *const c_char) -> NetConfig;
-}
-
-pub fn format_mac(mac: &[u8; 6]) -> String {
-    format!(
-        "{:02X}:{:02X}:{:02X}:{:02X}:{:02X}:{:02X}",
-        mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
-    )
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn format_mac_all_zero() {
-        assert_eq!(format_mac(&[0; 6]), "00:00:00:00:00:00");
-    }
-
-    #[test]
-    fn format_mac_hex() {
-        let mac = [0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54];
-        assert_eq!(format_mac(&mac), "FE:DC:BA:98:76:54");
-    }
-}
diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs
deleted file mode 100644
index d67a739..0000000
--- a/host/start-vm/tests/vm_command-basic.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-use std::ffi::{OsStr, OsString};
-use std::fs::{create_dir_all, File};
-
-use start_vm::vm_command;
-use test_helper::TempDir;
-
-fn main() -> std::io::Result<()> {
-    let service_dir_parent = TempDir::new()?;
-    let service_dir = service_dir_parent.path().join("vm-testvm");
-
-    let vm_dir = TempDir::new()?;
-
-    let kernel_path = vm_dir.path().join("testvm/config/vmlinux");
-    let image_path = vm_dir.path().join("testvm/config/blk/root.img");
-
-    create_dir_all(image_path.parent().unwrap())?;
-    File::create(&kernel_path)?;
-    File::create(&image_path)?;
-
-    let command = vm_command(&service_dir, vm_dir.path(), 4).unwrap();
-    assert_eq!(command.get_program(), "cloud-hypervisor");
-
-    let mut expected_disk_arg = OsString::from("path=");
-    expected_disk_arg.push(image_path);
-    expected_disk_arg.push(",readonly=on");
-
-    let expected_args = vec![
-        OsStr::new("--api-socket"),
-        OsStr::new("fd=4"),
-        OsStr::new("--cmdline"),
-        OsStr::new("console=ttyS0 root=PARTLABEL=root"),
-        OsStr::new("--memory"),
-        OsStr::new("size=256M,shared=on"),
-        OsStr::new("--console"),
-        OsStr::new("pty"),
-        OsStr::new("--kernel"),
-        kernel_path.as_os_str(),
-        OsStr::new("--disk"),
-        &expected_disk_arg,
-        OsStr::new("--serial"),
-        OsStr::new("file=/run/testvm.log"),
-    ];
-
-    assert!(command.get_args().eq(expected_args.into_iter()));
-    Ok(())
-}
diff --git a/host/start-vm/tests/vm_command-multiple-disks.rs b/host/start-vm/tests/vm_command-multiple-disks.rs
deleted file mode 100644
index 68ed495..0000000
--- a/host/start-vm/tests/vm_command-multiple-disks.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
-
-use std::collections::BTreeSet;
-use std::ffi::{OsStr, OsString};
-use std::fs::{create_dir, create_dir_all, File};
-use std::os::unix::fs::symlink;
-
-use start_vm::vm_command;
-use test_helper::TempDir;
-
-fn main() -> std::io::Result<()> {
-    let service_dir_parent = TempDir::new()?;
-    let service_dir = service_dir_parent.path().join("vm-testvm");
-
-    let vm_dir = TempDir::new()?;
-    let vm_config = vm_dir.path().join("testvm/config");
-
-    create_dir_all(&vm_config)?;
-    File::create(vm_config.join("vmlinux"))?;
-    create_dir(vm_config.join("blk"))?;
-
-    let image_paths: Vec<_> = (1..=2)
-        .map(|n| vm_config.join(format!("blk/disk{n}.img")))
-        .collect();
-
-    for image_path in &image_paths {
-        symlink("/dev/null", image_path)?;
-    }
-
-    let command = vm_command(&service_dir, vm_dir.path(), -1).unwrap();
-    let mut args = command.get_args();
-
-    assert!(args.any(|arg| arg == "--disk"));
-
-    let expected_disk_args = image_paths
-        .iter()
-        .map(|image_path| {
-            let mut expected_disk_arg = OsString::from("path=");
-            expected_disk_arg.push(image_path);
-            expected_disk_arg.push(",readonly=on");
-            expected_disk_arg
-        })
-        .collect::<BTreeSet<_>>();
-
-    let disk_args = args
-        .map(OsStr::to_os_string)
-        .take(expected_disk_args.len())
-        .collect::<BTreeSet<_>>();
-
-    assert_eq!(disk_args, expected_disk_args);
-
-    Ok(())
-}
diff --git a/host/start-vm/tests/vm_command-shared-dir.rs b/host/start-vm/tests/vm_command-shared-dir.rs
deleted file mode 100644
index 481230a..0000000
--- a/host/start-vm/tests/vm_command-shared-dir.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
-
-use std::collections::BTreeSet;
-use std::ffi::{OsStr, OsString};
-use std::fs::{create_dir, create_dir_all, File};
-use std::os::unix::fs::symlink;
-
-use start_vm::vm_command;
-use test_helper::TempDir;
-
-fn main() -> std::io::Result<()> {
-    let service_dir_parent = TempDir::new()?;
-    let service_dir = service_dir_parent.path().join("vm-testvm");
-
-    let vm_dir = TempDir::new()?;
-    let vm_config = vm_dir.path().join("testvm/config");
-
-    create_dir_all(&vm_config)?;
-    File::create(vm_config.join("vmlinux"))?;
-    create_dir(vm_config.join("blk"))?;
-    symlink("/dev/null", vm_config.join("blk/root.img"))?;
-
-    create_dir(vm_config.join("shared-dirs"))?;
-
-    create_dir(vm_config.join("shared-dirs/dir1"))?;
-    symlink("/", vm_config.join("shared-dirs/dir1/dir"))?;
-
-    create_dir(vm_config.join("shared-dirs/dir2"))?;
-    symlink("/", vm_config.join("shared-dirs/dir2/dir"))?;
-
-    let command = vm_command(&service_dir, vm_dir.path(), -1).unwrap();
-    let mut args = command.get_args();
-
-    assert!(args.any(|arg| arg == "--fs"));
-
-    let expected_fs_args = (1..=2)
-        .map(|i| format!("tag=dir{i},socket=../fs-testvm-dir{i}/env/virtiofsd.sock"))
-        .map(OsString::from)
-        .collect::<BTreeSet<_>>();
-
-    let fs_args = args
-        .map(OsStr::to_os_string)
-        .take(expected_fs_args.len())
-        .collect::<BTreeSet<_>>();
-
-    assert_eq!(fs_args, expected_fs_args);
-
-    Ok(())
-}
diff --git a/host/start-vm/ch.h b/host/start-vmm/ch.h
index 9007153..5143723 100644
--- a/host/start-vm/ch.h
+++ b/host/start-vmm/ch.h
@@ -1,11 +1,16 @@
 // SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
 
 #include <stdint.h>
 
 struct ch_device;
 
-int ch_add_net(const char *vm_name, int tap, const uint8_t mac[6],
+struct net_config {
+	int fd;
+	uint8_t mac[6];
+};
+
+int ch_add_net(const char *vm_name, const struct net_config *,
                struct ch_device **out);
 int ch_remove_device(const char *vm_name, struct ch_device *);
 
diff --git a/host/start-vmm/ch.rs b/host/start-vmm/ch.rs
new file mode 100644
index 0000000..0ef1354
--- /dev/null
+++ b/host/start-vmm/ch.rs
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+use std::ffi::{CStr, OsStr, OsString};
+use std::io::Write;
+use std::mem::take;
+use std::num::NonZeroI32;
+use std::os::raw::{c_char, c_int};
+use std::os::unix::prelude::*;
+use std::process::{Command, Stdio};
+
+use miniserde::{json, Serialize};
+
+use crate::net::MacAddress;
+
+// Trivially safe.
+const EINVAL: NonZeroI32 = unsafe { NonZeroI32::new_unchecked(22) };
+const EPERM: NonZeroI32 = unsafe { NonZeroI32::new_unchecked(1) };
+const EPROTO: NonZeroI32 = unsafe { NonZeroI32::new_unchecked(71) };
+
+#[derive(Serialize)]
+pub struct ConsoleConfig {
+    pub mode: &'static str,
+    pub file: Option<String>,
+}
+
+#[derive(Serialize)]
+pub struct DiskConfig {
+    pub path: String,
+    pub readonly: bool,
+}
+
+#[derive(Serialize)]
+pub struct FsConfig {
+    pub socket: String,
+    pub tag: String,
+}
+
+#[derive(Serialize)]
+pub struct GpuConfig {
+    pub socket: String,
+}
+
+#[derive(Serialize)]
+#[repr(C)]
+pub struct NetConfig {
+    pub fd: RawFd,
+    pub mac: MacAddress,
+}
+
+#[derive(Serialize)]
+pub struct MemoryConfig {
+    pub size: i64,
+    pub shared: bool,
+}
+
+#[derive(Serialize)]
+pub struct PayloadConfig {
+    pub kernel: String,
+    pub cmdline: &'static str,
+}
+
+#[derive(Serialize)]
+pub struct VmConfig {
+    pub console: ConsoleConfig,
+    pub disks: Vec<DiskConfig>,
+    pub fs: Vec<FsConfig>,
+    pub gpu: Vec<GpuConfig>,
+    pub memory: MemoryConfig,
+    pub net: Vec<NetConfig>,
+    pub payload: PayloadConfig,
+    pub serial: ConsoleConfig,
+}
+
+fn command(vm_name: &str, s: impl AsRef<OsStr>) -> Command {
+    let mut api_socket_path = OsString::from("/run/service/ext-vm-");
+    api_socket_path.push(vm_name);
+    api_socket_path.push("/env/cloud-hypervisor.sock");
+
+    let mut command = Command::new("ch-remote");
+    command.stdin(Stdio::null());
+    command.arg("--api-socket");
+    command.arg(api_socket_path);
+    command.arg(s);
+    command
+}
+
+pub fn create_vm(vm_name: &str, mut config: VmConfig) -> Result<(), String> {
+    // Net devices can't be created from file descriptors in vm.create.
+    // https://github.com/cloud-hypervisor/cloud-hypervisor/issues/5523
+    let nets = take(&mut config.net);
+
+    let mut ch_remote = command(vm_name, "create")
+        .args(["--", "-"])
+        .stdin(Stdio::piped())
+        .spawn()
+        .map_err(|e| format!("failed to start ch-remote: {e}"))?;
+
+    let json = json::to_string(&config);
+    write!(ch_remote.stdin.as_ref().unwrap(), "{}", json)
+        .map_err(|e| format!("writing to ch-remote's stdin: {e}"))?;
+
+    let status = ch_remote
+        .wait()
+        .map_err(|e| format!("waiting for ch-remote: {e}"))?;
+    if status.success() {
+    } else if let Some(code) = status.code() {
+        return Err(format!("ch-remote exited {code}"));
+    } else {
+        let signal = status.signal().unwrap();
+        return Err(format!("ch-remote killed by signal {signal}"));
+    }
+
+    for net in nets {
+        add_net(vm_name, &net).map_err(|e| format!("failed to add net: {e}"))?;
+    }
+
+    Ok(())
+}
+
+pub fn add_net(vm_name: &str, net: &NetConfig) -> Result<OsString, NonZeroI32> {
+    let mut ch_remote = command(vm_name, "add-net")
+        .arg(format!("fd={},mac={}", net.fd, net.mac))
+        .stdout(Stdio::piped())
+        .spawn()
+        .or(Err(EPERM))?;
+
+    let jq_out = match Command::new("jq")
+        .args(["-j", ".id"])
+        .stdin(ch_remote.stdout.take().unwrap())
+        .stderr(Stdio::inherit())
+        .output()
+    {
+        Ok(o) => o,
+        Err(_) => {
+            // Try not to leave a zombie.
+            let _ = ch_remote.kill();
+            let _ = ch_remote.wait();
+            return Err(EPERM);
+        }
+    };
+
+    if let Ok(ch_remote_status) = ch_remote.wait() {
+        if ch_remote_status.success() && jq_out.status.success() {
+            return Ok(OsString::from_vec(jq_out.stdout));
+        }
+    }
+
+    Err(EPROTO)
+}
+
+pub fn remove_device(vm_name: &str, device_id: &OsStr) -> Result<(), NonZeroI32> {
+    let ch_remote = command(vm_name, "remove-device")
+        .arg(device_id)
+        .status()
+        .or(Err(EPERM))?;
+
+    if ch_remote.success() {
+        Ok(())
+    } else {
+        Err(EPROTO)
+    }
+}
+
+/// # Safety
+///
+/// - `vm_name` must point to a valid C string.
+/// - `tap` must be a file descriptor describing an tap device.
+/// - `mac` must be a valid pointer.
+#[export_name = "ch_add_net"]
+unsafe extern "C" fn add_net_c(
+    vm_name: *const c_char,
+    net: &NetConfig,
+    id: *mut *mut OsString,
+) -> c_int {
+    let Ok(vm_name) = CStr::from_ptr(vm_name).to_str() else {
+        return EINVAL.into();
+    };
+
+    match add_net(vm_name, net) {
+        Err(e) => e.get(),
+        Ok(id_str) => {
+            if !id.is_null() {
+                let token = Box::into_raw(Box::new(id_str));
+                *id = token;
+            }
+            0
+        }
+    }
+}
+
+/// # Safety
+///
+/// - `vm_name` must point to a valid C string.
+/// - `id` must be a device ID obtained by calling `add_net_c`.  After
+///   calling `remove_device_c`, the pointer is no longer valid.
+#[export_name = "ch_remove_device"]
+unsafe extern "C" fn remove_device_c(vm_name: *const c_char, device_id: *mut OsString) -> c_int {
+    let Ok(vm_name) = CStr::from_ptr(vm_name).to_str() else {
+        return EINVAL.into();
+    };
+    let device_id = Box::from_raw(device_id);
+
+    if let Err(e) = remove_device(vm_name, device_id.as_ref()) {
+        e.get()
+    } else {
+        0
+    }
+}
+
+/// # Safety
+///
+/// `id` must be a device ID obtained by calling `add_net_c`.  After
+/// calling `device_free`, the pointer is no longer valid.
+#[export_name = "ch_device_free"]
+unsafe extern "C" fn device_free(id: *mut OsString) {
+    if !id.is_null() {
+        drop(Box::from_raw(id))
+    }
+}
diff --git a/host/start-vmm/default.nix b/host/start-vmm/default.nix
new file mode 100644
index 0000000..15e1211
--- /dev/null
+++ b/host/start-vmm/default.nix
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+import ../../lib/call-package.nix (
+{ src, lib, stdenv, fetchCrate, fetchFromGitHub, fetchurl, buildPackages
+, meson, ninja, rustc, clippy, run-spectrum-vm
+}:
+
+let
+  packageCache = [
+    (fetchCrate {
+      pname = "itoa";
+      version = "1.0.10";
+      unpack = false;
+      hash = "sha256-saRtGhcdhlql+D+SaVdlyqBHqbTLriy/N9vWE6eT/Uw=";
+    })
+    (fetchurl {
+      name = "miniserde-0.1.37.tar.gz";
+      url = "https://github.com/dtolnay/miniserde/archive/0.1.37.tar.gz";
+      hash = "sha256-zE4WY6uI/7P7NaJyb2aZAnSL1rBSq5hAtx26151qUEk=";
+    })
+    (fetchCrate {
+      pname = "proc-macro2";
+      version = "1.0.78";
+      unpack = false;
+      hash = "sha256-4kIq1kXYnJn48+a4ip/eyn+r6sg2sQAjccQ2fI+YSq4=";
+    })
+    (fetchCrate {
+      pname = "quote";
+      version = "1.0.35";
+      unpack = false;
+      hash = "sha256-KR7Jq179k0qvUDpkZsXVJRU10QjudHRyw5d8xazIaO8=";
+    })
+    (fetchCrate {
+      pname = "ryu";
+      version = "1.0.17";
+      unpack = false;
+      hash = "sha256-6GaXyRYBmoWIyZtfrDzq107AtLgZcHpoL9TSP6DOG6E=";
+    })
+    (fetchCrate {
+      pname = "syn";
+      version = "2.0.41";
+      unpack = false;
+      hash = "sha256-RMiyjEd8w78OeWZWHjRgEw4SVfehz3GTEHXxxeen4mk=";
+    })
+    (fetchCrate {
+      pname = "unicode-ident";
+      version = "1.0.12";
+      unpack = false;
+      hash = "sha256-M1S5rD+uH/Z1XLbbU2g622YWNPZ1V5Qt6k+s6+wP7ks=";
+    })
+  ];
+in
+
+stdenv.mkDerivation (finalAttrs: {
+  name = "start-vmm";
+
+  src = lib.fileset.toSource {
+    root = ../..;
+    fileset = lib.fileset.intersection src ./.;
+  };
+  sourceRoot = "source/host/start-vmm";
+
+  depsBuildBuild = [ buildPackages.stdenv.cc ];
+  nativeBuildInputs = [ meson ninja rustc ];
+
+  postPatch = lib.concatMapStringsSep "\n" (crate: ''
+    mkdir -p subprojects/packagecache
+    ln -s ${crate} subprojects/packagecache/${crate.name}
+  '') packageCache;
+
+  preConfigure = ''
+    mesonFlagsArray+=(-Drust_args="-C panic=abort" -Dtests=false -Dwerror=true)
+  '';
+
+  passthru.tests = {
+    clippy = finalAttrs.finalPackage.overrideAttrs (
+      { name, nativeBuildInputs ? [], ... }:
+      {
+        name = "${name}-clippy";
+        nativeBuildInputs = nativeBuildInputs ++ [ clippy ];
+        RUSTC = "clippy-driver";
+        preConfigure = ''
+          # It's not currently possible to enable warnings only for
+          # non-subprojects without enumerating the subprojects.
+          # https://github.com/mesonbuild/meson/issues/9398#issuecomment-954094750
+          mesonFlagsArray+=(
+              -Dwerror=true
+              -Dproc-macro2:werror=false
+              -Dproc-macro2:warning_level=0
+          )
+        '';
+        postBuild = ''touch $out && exit 0'';
+      }
+    );
+
+    run = run-spectrum-vm.override { start-vmm = finalAttrs.finalPackage; };
+
+    tests = finalAttrs.finalPackage.overrideAttrs ({ name, ... }: {
+      name = "${name}-tests";
+      preConfigure = "";
+      doCheck = true;
+    });
+  };
+
+  meta = {
+    mainProgram = "start-vmm";
+  };
+})
+) (_: {})
diff --git a/host/start-vmm/fork.c b/host/start-vmm/fork.c
new file mode 100644
index 0000000..2ec17c2
--- /dev/null
+++ b/host/start-vmm/fork.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <sys/wait.h>
+
+// Positive return value: in grandparent, pid of grandchild.
+// 0: in grandchild.
+// Negative return value: errno.
+int double_fork(void)
+{
+	int fd[2], v;
+	size_t acc = 0;
+	ssize_t r;
+	pid_t child;
+
+	if (pipe(fd) == -1)
+		return -1;
+
+	switch (child = fork()) {
+	case -1:
+		close(fd[0]);
+		close(fd[1]);
+		return -1;
+	case 0:
+		close(fd[0]);
+		switch ((v = fork())) {
+	        case 0:
+			close(fd[1]);
+			return 0;
+		case -1:
+			v = -errno;
+			[[fallthrough]];
+		default:
+			do {
+				r = write(fd[1], (char *)&v + acc, sizeof v - acc);
+			} while ((r != -1 || errno == EINTR) && (acc += r) < sizeof v);
+			exit(v < 0 || r == -1);
+		}
+	default:
+		close(fd[1]);
+		do {
+			r = read(fd[0], (char *)&v + acc, sizeof v - acc);
+		} while ((r != -1 || errno == EINTR) && (acc += r) < sizeof v);
+		close(fd[0]);
+		if (r == -1) {
+			kill(child, SIGKILL);
+			waitpid(child, NULL, 0);
+		}
+		return v;
+	}
+}
diff --git a/host/start-vmm/fork.rs b/host/start-vmm/fork.rs
new file mode 100644
index 0000000..3c0e11a
--- /dev/null
+++ b/host/start-vmm/fork.rs
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+use std::ffi::c_int;
+
+extern "C" {
+    pub fn double_fork() -> c_int;
+}
diff --git a/host/start-vmm/lib.rs b/host/start-vmm/lib.rs
new file mode 100644
index 0000000..7dd358a
--- /dev/null
+++ b/host/start-vmm/lib.rs
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+mod ch;
+mod fork;
+mod net;
+mod s6;
+mod unix;
+
+use std::borrow::Cow;
+use std::env::args_os;
+use std::ffi::{CString, OsStr};
+use std::fs::remove_file;
+use std::io::{self, ErrorKind};
+use std::os::unix::net::UnixListener;
+use std::os::unix::prelude::*;
+use std::os::unix::process::parent_id;
+use std::path::Path;
+use std::process::{exit, Command};
+
+use ch::{ConsoleConfig, DiskConfig, FsConfig, GpuConfig, MemoryConfig, PayloadConfig, VmConfig};
+use fork::double_fork;
+use net::net_setup;
+use s6::notify_readiness;
+use unix::clear_cloexec;
+
+const SIGTERM: i32 = 15;
+
+extern "C" {
+    fn kill(pid: i32, sig: i32) -> i32;
+}
+
+pub fn prog_name() -> String {
+    args_os()
+        .next()
+        .as_ref()
+        .map(Path::new)
+        .and_then(Path::file_name)
+        .map(OsStr::to_string_lossy)
+        .unwrap_or(Cow::Borrowed("start-vmm"))
+        .into_owned()
+}
+
+pub fn create_api_socket() -> Result<UnixListener, String> {
+    let _ = remove_file("env/cloud-hypervisor.sock");
+    let api_socket = UnixListener::bind("env/cloud-hypervisor.sock")
+        .map_err(|e| format!("creating API socket: {e}"))?;
+
+    // Safe because we own api_socket.
+    if unsafe { clear_cloexec(api_socket.as_fd()) } == -1 {
+        let errno = io::Error::last_os_error();
+        return Err(format!("clearing CLOEXEC on API socket fd: {}", errno));
+    }
+
+    Ok(api_socket)
+}
+
+pub fn vm_config(vm_name: &str, config_root: &Path) -> Result<VmConfig, String> {
+    if config_root.to_str().is_none() {
+        return Err(format!("config root {:?} is not valid UTF-8", config_root));
+    }
+
+    let config_dir = config_root.join(vm_name).join("config");
+
+    let blk_dir = config_dir.join("blk");
+    let kernel_path = config_dir.join("vmlinux");
+    let net_providers_dir = config_dir.join("providers/net");
+    let shared_dirs_dir = config_dir.join("shared-dirs");
+    let wayland_path = config_dir.join("wayland");
+
+    Ok(VmConfig {
+        console: ConsoleConfig {
+            mode: "Pty",
+            file: None,
+        },
+        disks: match blk_dir.read_dir() {
+            Ok(entries) => entries
+                .into_iter()
+                .map(|result| {
+                    Ok(result
+                        .map_err(|e| format!("examining directory entry: {e}"))?
+                        .path())
+                })
+                .filter(|result| {
+                    result
+                        .as_ref()
+                        .map(|entry| entry.extension() == Some(OsStr::new("img")))
+                        .unwrap_or(true)
+                })
+                .map(|result: Result<_, String>| {
+                    let entry = result?.to_str().unwrap().to_string();
+
+                    if entry.contains(',') {
+                        return Err(format!("illegal ',' character in path {:?}", entry));
+                    }
+
+                    Ok(DiskConfig {
+                        path: entry,
+                        readonly: true,
+                    })
+                })
+                .collect::<Result<_, _>>()?,
+            Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)),
+        },
+        fs: match shared_dirs_dir.read_dir() {
+            Ok(entries) => entries
+                .into_iter()
+                .map(|result| {
+                    let entry = result
+                        .map_err(|e| format!("examining directory entry: {}", e))?
+                        .file_name();
+
+                    let entry = entry.to_str().ok_or_else(|| {
+                        format!("shared directory name {:?} is not valid UTF-8", entry)
+                    })?;
+
+                    Ok(FsConfig {
+                        tag: entry.to_string(),
+                        socket: format!("../fs-{vm_name}-{entry}/env/virtiofsd.sock"),
+                    })
+                })
+                .collect::<Result<_, String>>()?,
+            Err(e) if e.kind() == ErrorKind::NotFound => Default::default(),
+            Err(e) => return Err(format!("reading directory {:?}: {e}", shared_dirs_dir)),
+        },
+        gpu: match wayland_path.try_exists() {
+            Ok(true) => vec![GpuConfig {
+                socket: format!("../gpu-{vm_name}/env/crosvm.sock"),
+            }],
+            Ok(false) => vec![],
+            Err(e) => return Err(format!("checking for existence of {:?}: {e}", wayland_path)),
+        },
+        memory: MemoryConfig {
+            size: 256 << 20,
+            shared: true,
+        },
+        net: match net_providers_dir.read_dir() {
+            Ok(entries) => entries
+                .into_iter()
+                .map(|result| {
+                    let entry = result
+                        .map_err(|e| format!("examining directory entry: {}", e))?
+                        .file_name();
+
+                    // Safe because provider_name is the name of a directory entry, so
+                    // can't contain a null byte.
+                    let provider_name = unsafe { CString::from_vec_unchecked(entry.into_vec()) };
+
+                    // Safe because we pass a valid pointer and check the result.
+                    let net = unsafe { net_setup(provider_name.as_ptr()) };
+                    if net.fd == -1 {
+                        let e = io::Error::last_os_error();
+                        return Err(format!("setting up networking failed: {e}"));
+                    }
+
+                    Ok(net)
+                })
+                // TODO: to support multiple net providers, we'll need
+                // a better naming scheme for tap and bridge devices.
+                .take(1)
+                .collect::<Result<_, _>>()?,
+            Err(e) if e.kind() == ErrorKind::NotFound => Default::default(),
+            Err(e) => return Err(format!("reading directory {:?}: {e}", net_providers_dir)),
+        },
+        payload: PayloadConfig {
+            kernel: kernel_path.to_str().unwrap().to_string(),
+            cmdline: "console=ttyS0 root=PARTLABEL=root",
+        },
+        serial: ConsoleConfig {
+            mode: "File",
+            file: Some(format!("/run/{vm_name}.log")),
+        },
+    })
+}
+
+/// # Safety
+///
+/// Calls [notify_readiness], so can only be called once per process.
+unsafe fn create_vm_child_main(vm_name: &str, config: VmConfig) -> ! {
+    if let Err(e) = ch::create_vm(vm_name, config) {
+        eprintln!("{}: creating VM: {e}", prog_name());
+        if kill(parent_id() as _, SIGTERM) == -1 {
+            let e = io::Error::last_os_error();
+            eprintln!("{}: killing cloud-hypervisor: {e}", prog_name());
+        };
+        exit(1);
+    }
+
+    if let Err(e) = notify_readiness() {
+        eprintln!("{}: failed to notify readiness: {e}", prog_name());
+        exit(1);
+    }
+
+    exit(0)
+}
+
+pub fn create_vm(dir: &Path, config_root: &Path) -> Result<(), String> {
+    let vm_name = dir
+        .file_name()
+        .ok_or_else(|| "directory has no name".to_string())?;
+
+    let vm_name = &vm_name
+        .to_str()
+        .ok_or_else(|| format!("VM name {:?} is not valid UTF-8", vm_name))?;
+
+    if !vm_name.starts_with("vm-") {
+        return Err("not running from a VM service directory".to_string());
+    }
+
+    if vm_name.contains(',') {
+        return Err(format!("VM name may not contain a comma: {:?}", vm_name));
+    }
+
+    let vm_name = &vm_name[3..];
+    let config = vm_config(vm_name, config_root)?;
+
+    // SAFETY: safe because we ensure we don't violate any invariants
+    // concerning OS resources shared between processes, by only
+    // passing data structs to the child main function.
+    match unsafe { double_fork() } {
+        e if e < 0 => Err(format!("double fork: {}", io::Error::from_raw_os_error(-e))),
+        // SAFETY: create_vm_child_main can only be called once per process,
+        // but this is a new process, so we know it hasn't been called before.
+        0 => unsafe { create_vm_child_main(vm_name, config) },
+        _ => Ok(()),
+    }
+}
+
+pub fn vm_command(api_socket_fd: RawFd) -> Result<Command, String> {
+    let mut command = Command::new("cloud-hypervisor");
+    command.args(["--api-socket", &format!("fd={api_socket_fd}")]);
+
+    Ok(command)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_vm_name_comma() {
+        let e = create_vm(Path::new("/vm-,"), Path::new("/")).unwrap_err();
+        assert!(e.contains("comma"), "unexpected error: {:?}", e);
+    }
+}
diff --git a/host/start-vmm/meson.build b/host/start-vmm/meson.build
new file mode 100644
index 0000000..564be1b
--- /dev/null
+++ b/host/start-vmm/meson.build
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: EUPL-1.2+
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+project('start-vmm', 'rust', 'c',
+  default_options : ['c_std=c2x', 'rust_std=2018'])
+
+add_project_arguments('-D_GNU_SOURCE', '-Wno-error=attributes', language : 'c')
+
+miniserde_dep = dependency('miniserde')
+
+c_lib = static_library('start-vmm', 'fork.c', 'net.c', 'net-util.c', 'unix.c')
+rust_lib = static_library('start_vmm', 'lib.rs',
+  dependencies : miniserde_dep,
+  link_with : c_lib)
+
+rust_lib_dep = declare_dependency(
+  dependencies : miniserde_dep,
+  link_with : rust_lib)
+
+executable('start-vmm', 'start-vmm.rs',
+  dependencies : miniserde_dep,
+  link_with : rust_lib,
+  install : true)
+
+if get_option('tests')
+  test_exe = executable('start-vmm-test', 'lib.rs',
+    dependencies : miniserde_dep,
+    rust_args : ['--test'],
+    link_with : c_lib)
+  test('Rust unit tests', test_exe, protocol : 'rust')
+
+  subdir('tests')
+endif
diff --git a/host/start-vmm/meson_options.txt b/host/start-vmm/meson_options.txt
new file mode 100644
index 0000000..21844bd
--- /dev/null
+++ b/host/start-vmm/meson_options.txt
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: EUPL-1.2+
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+option('tests',
+  type : 'boolean',
+  description : 'Build the tests')
diff --git a/host/start-vm/net-util.c b/host/start-vmm/net-util.c
index 1d2fb33..1d2fb33 100644
--- a/host/start-vm/net-util.c
+++ b/host/start-vmm/net-util.c
diff --git a/host/start-vm/net-util.h b/host/start-vmm/net-util.h
index 5ec09c2..5ec09c2 100644
--- a/host/start-vm/net-util.h
+++ b/host/start-vmm/net-util.h
diff --git a/host/start-vm/net.c b/host/start-vmm/net.c
index c8409f4..4fc5486 100644
--- a/host/start-vm/net.c
+++ b/host/start-vmm/net.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
 
 #include "ch.h"
 #include "net-util.h"
@@ -51,12 +51,15 @@ static int client_net_setup(const char *bridge_name)
 static int router_net_setup(const char *bridge_name, const char *router_vm_name,
                             const uint8_t mac[6], struct ch_device **out)
 {
-	int e, fd = setup_tap(bridge_name, "router");
-	if (fd == -1)
+	struct net_config net;
+	int e;
+
+	memcpy(&net.mac, mac, sizeof net.mac);
+	if ((net.fd = setup_tap(bridge_name, "router")) == -1)
 		return -1;
 
-	e = ch_add_net(router_vm_name, fd, mac, out);
-	close(fd);
+	e = ch_add_net(router_vm_name, &net, out);
+	close(net.fd);
 	if (!e)
 		return 0;
 	errno = e;
@@ -149,11 +152,6 @@ static int exit_listener_setup(const char *router_vm_name,
 	}
 }
 
-struct net_config {
-	int fd;
-	char mac[6];
-};
-
 struct net_config net_setup(const char *router_vm_name)
 {
 	struct ch_device *router_vm_net_device = NULL;
diff --git a/host/start-vmm/net.rs b/host/start-vmm/net.rs
new file mode 100644
index 0000000..8ca19c5
--- /dev/null
+++ b/host/start-vmm/net.rs
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+use std::borrow::Cow;
+use std::fmt::{self, Display, Formatter};
+use std::os::raw::c_char;
+
+use miniserde::ser::Fragment;
+use miniserde::Serialize;
+
+use crate::ch::NetConfig;
+
+#[repr(transparent)]
+pub struct MacAddress([u8; 6]);
+
+impl MacAddress {
+    pub fn new(octets: [u8; 6]) -> Self {
+        Self(octets)
+    }
+}
+
+impl Display for MacAddress {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{:02X}:{:02X}:{:02X}:{:02X}:{:02X}:{:02X}",
+            self.0[0], self.0[1], self.0[2], self.0[3], self.0[4], self.0[5]
+        )
+    }
+}
+
+impl Serialize for MacAddress {
+    fn begin(&self) -> Fragment {
+        Fragment::Str(Cow::Owned(self.to_string()))
+    }
+}
+
+extern "C" {
+    pub fn net_setup(provider_vm_name: *const c_char) -> NetConfig;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn mac_to_string_all_zero() {
+        assert_eq!(MacAddress([0; 6]).to_string(), "00:00:00:00:00:00");
+    }
+
+    #[test]
+    fn mac_to_string_hex() {
+        let mac = MacAddress([0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54]);
+        assert_eq!(mac.to_string(), "FE:DC:BA:98:76:54");
+    }
+}
diff --git a/host/start-vm/s6.rs b/host/start-vmm/s6.rs
index eda587f..eda587f 100644
--- a/host/start-vm/s6.rs
+++ b/host/start-vmm/s6.rs
diff --git a/host/start-vm/shell.nix b/host/start-vmm/shell.nix
index ed1a190..ed1a190 100644
--- a/host/start-vm/shell.nix
+++ b/host/start-vmm/shell.nix
diff --git a/host/start-vm/start-vm.rs b/host/start-vmm/start-vmm.rs
index 7dfca09..5479b5f 100644
--- a/host/start-vm/start-vm.rs
+++ b/host/start-vmm/start-vmm.rs
@@ -6,7 +6,7 @@ use std::os::unix::prelude::*;
 use std::path::Path;
 use std::process::exit;
 
-use start_vm::{create_api_socket, notify_readiness, prog_name, vm_command};
+use start_vmm::{create_api_socket, create_vm, prog_name, vm_command};
 
 /// # Safety
 ///
@@ -22,11 +22,11 @@ unsafe fn run() -> String {
         Err(e) => return e,
     };
 
-    if let Err(e) = notify_readiness() {
+    if let Err(e) = create_vm(&dir, Path::new("/run/vm")) {
         return e;
     }
 
-    match vm_command(&dir, Path::new("/run/vm"), api_socket.into_raw_fd()) {
+    match vm_command(api_socket.into_raw_fd()) {
         Ok(mut command) => format!("failed to exec: {}", command.exec()),
         Err(e) => e,
     }
diff --git a/host/start-vmm/subprojects/itoa.wrap b/host/start-vmm/subprojects/itoa.wrap
new file mode 100644
index 0000000..ce662df
--- /dev/null
+++ b/host/start-vmm/subprojects/itoa.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = itoa-1.0.10
+source_url = https://crates.io/api/v1/crates/itoa/1.0.10/download
+source_filename = itoa-1.0.10.tar.gz
+source_hash = b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c
+patch_directory = itoa
diff --git a/host/start-vmm/subprojects/miniserde.wrap b/host/start-vmm/subprojects/miniserde.wrap
new file mode 100644
index 0000000..1fcbb87
--- /dev/null
+++ b/host/start-vmm/subprojects/miniserde.wrap
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+# The GitHub archive is used so that mini_internal is included,
+# and can be built with the same meson.build,
+# to avoid exposing it as a dependency.
+[wrap-file]
+directory = miniserde-0.1.37
+source_url = https://github.com/dtolnay/miniserde/archive/0.1.37.tar.gz
+source_filename = miniserde-0.1.37.tar.gz
+source_hash = cc4e1663ab88ffb3fb35a2726f669902748bd6b052ab9840b71dbad79d6a5049
+depth = 1
+patch_directory = miniserde
diff --git a/host/start-vmm/subprojects/packagefiles/itoa/meson.build b/host/start-vmm/subprojects/packagefiles/itoa/meson.build
new file mode 100644
index 0000000..eeb2cf6
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/itoa/meson.build
@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('itoa', 'rust', version : '1.0.10', default_options : ['rust_std=2018'])
+
+itoa = static_library('itoa', 'src/lib.rs', rust_crate_type : 'rlib')
+
+itoa_dep = declare_dependency(link_with : itoa)
+
+meson.override_dependency('itoa', itoa_dep)
diff --git a/host/start-vmm/subprojects/packagefiles/miniserde/meson.build b/host/start-vmm/subprojects/packagefiles/miniserde/meson.build
new file mode 100644
index 0000000..e138d8b
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/miniserde/meson.build
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('miniserde', 'rust', version : '0.1.37',
+  default_options : ['build.rust_std=2021', 'rust_std=2021'])
+
+quote_dep = dependency('quote', native : true)
+syn_dep = dependency('syn', native : true)
+
+mini_internal = shared_library('mini_internal', 'derive/src/lib.rs',
+  dependencies : [quote_dep, syn_dep],
+  native : true,
+  rust_args : ['-C', 'panic=unwind'],
+  rust_crate_type : 'proc-macro')
+
+itoa_dep = dependency('itoa')
+ryu_dep = dependency('ryu')
+
+miniserde = static_library('miniserde', 'src/lib.rs',
+  dependencies : [itoa_dep, ryu_dep],
+  link_with : mini_internal,
+  rust_crate_type : 'rlib')
+
+miniserde_dep = declare_dependency(
+  dependencies : [itoa_dep, ryu_dep],
+  link_with : miniserde)
+
+meson.override_dependency('miniserde', miniserde_dep)
diff --git a/host/start-vmm/subprojects/packagefiles/proc-macro2/meson.build b/host/start-vmm/subprojects/packagefiles/proc-macro2/meson.build
new file mode 100644
index 0000000..1e29ed4
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/proc-macro2/meson.build
@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('proc-macro2', 'rust', version : '1.0.78',
+  default_options : ['build.rust_std=2021', 'rust_std=2021'])
+
+unicode_ident_dep = dependency('unicode-ident', native : true)
+
+proc_macro2 = static_library('proc_macro2', 'src/lib.rs',
+  dependencies : unicode_ident_dep,
+  native : true,
+  rust_args : ['-C', 'panic=unwind', '--cfg', 'feature="proc-macro"'],
+  rust_crate_type : 'rlib')
+
+proc_macro2_dep = declare_dependency(
+  dependencies : unicode_ident_dep,
+  link_with : proc_macro2)
+
+meson.override_dependency('proc-macro2', proc_macro2_dep, native : true)
diff --git a/host/start-vmm/subprojects/packagefiles/quote/meson.build b/host/start-vmm/subprojects/packagefiles/quote/meson.build
new file mode 100644
index 0000000..4a003d8
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/quote/meson.build
@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('quote', 'rust', version : '1.0.35',
+  default_options : ['build.rust_std=2018', 'rust_std=2018'])
+
+proc_macro2_dep = dependency('proc-macro2', native : true)
+
+quote = static_library('quote', 'src/lib.rs',
+  dependencies : proc_macro2_dep,
+  native : true,
+  rust_args : ['-C', 'panic=unwind'],
+  rust_crate_type : 'rlib')
+
+quote_dep = declare_dependency(
+  dependencies : proc_macro2_dep,
+  link_with : quote)
+
+meson.override_dependency('quote', quote_dep, native : true)
diff --git a/host/start-vmm/subprojects/packagefiles/ryu/meson.build b/host/start-vmm/subprojects/packagefiles/ryu/meson.build
new file mode 100644
index 0000000..a7ca612
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/ryu/meson.build
@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('ryu', 'rust', version : '1.0.17', default_options : ['rust_std=2018'])
+
+ryu = static_library('ryu', 'src/lib.rs', rust_crate_type : 'rlib')
+
+ryu_dep = declare_dependency(link_with : ryu)
+
+meson.override_dependency('ryu', ryu_dep)
diff --git a/host/start-vmm/subprojects/packagefiles/syn/meson.build b/host/start-vmm/subprojects/packagefiles/syn/meson.build
new file mode 100644
index 0000000..60ea88f
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/syn/meson.build
@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('syn', 'rust', version : '2.0.41',
+  default_options : ['build.rust_std=2021', 'rust_std=2021'])
+
+proc_macro2_dep = dependency('proc-macro2', native : true)
+quote_dep = dependency('quote', native : true)
+
+syn = static_library('syn', 'src/lib.rs',
+  dependencies : [proc_macro2_dep, quote_dep],
+  native : true,
+  rust_args : [
+    '-C', 'panic=unwind',
+    '--cfg', 'feature="clone-impls"',
+    '--cfg', 'feature="derive"',
+    '--cfg', 'feature="parsing"',
+    '--cfg', 'feature="printing"',
+    '--cfg', 'feature="proc-macro"',
+  ],
+  rust_crate_type : 'rlib')
+
+syn_dep = declare_dependency(
+  dependencies : [proc_macro2_dep, quote_dep],
+  link_with : syn)
+
+meson.override_dependency('syn', syn_dep, native : true)
diff --git a/host/start-vmm/subprojects/packagefiles/unicode-ident/meson.build b/host/start-vmm/subprojects/packagefiles/unicode-ident/meson.build
new file mode 100644
index 0000000..7146ec4
--- /dev/null
+++ b/host/start-vmm/subprojects/packagefiles/unicode-ident/meson.build
@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+# SPDX-License-Identifier: MIT
+
+project('unicode-ident', 'rust', version : '1.0.9',
+  default_options : ['build.rust_std=2018', 'rust_std=2018'])
+
+unicode_ident = static_library('unicode_ident', 'src/lib.rs',
+  native : true,
+  rust_args : ['-C', 'panic=unwind'],
+  rust_crate_type : 'rlib')
+
+unicode_ident_dep = declare_dependency(link_with : unicode_ident)
+
+meson.override_dependency('unicode-ident', unicode_ident_dep, native : true)
diff --git a/host/start-vmm/subprojects/proc-macro2.wrap b/host/start-vmm/subprojects/proc-macro2.wrap
new file mode 100644
index 0000000..f8dd29e
--- /dev/null
+++ b/host/start-vmm/subprojects/proc-macro2.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = proc-macro2-1.0.78
+source_url = https://crates.io/api/v1/crates/proc-macro2/1.0.78/download
+source_filename = proc-macro2-1.0.78.tar.gz
+source_hash = e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae
+patch_directory = proc-macro2
diff --git a/host/start-vmm/subprojects/quote.wrap b/host/start-vmm/subprojects/quote.wrap
new file mode 100644
index 0000000..cdbd3cc
--- /dev/null
+++ b/host/start-vmm/subprojects/quote.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = quote-1.0.35
+source_url = https://crates.io/api/v1/crates/quote/1.0.35/download
+source_filename = quote-1.0.35.tar.gz
+source_hash = 291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef
+patch_directory = quote
diff --git a/host/start-vmm/subprojects/ryu.wrap b/host/start-vmm/subprojects/ryu.wrap
new file mode 100644
index 0000000..07fa27f
--- /dev/null
+++ b/host/start-vmm/subprojects/ryu.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = ryu-1.0.17
+source_url = https://crates.io/api/v1/crates/ryu/1.0.17/download
+source_filename = ryu-1.0.17.tar.gz
+source_hash = e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1
+patch_directory = ryu
diff --git a/host/start-vmm/subprojects/syn.wrap b/host/start-vmm/subprojects/syn.wrap
new file mode 100644
index 0000000..f01014a
--- /dev/null
+++ b/host/start-vmm/subprojects/syn.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = syn-2.0.41
+source_url = https://crates.io/api/v1/crates/syn/2.0.41/download
+source_filename = syn-2.0.41.tar.gz
+source_hash = 44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269
+patch_directory = syn
diff --git a/host/start-vmm/subprojects/unicode-ident.wrap b/host/start-vmm/subprojects/unicode-ident.wrap
new file mode 100644
index 0000000..e6ed206
--- /dev/null
+++ b/host/start-vmm/subprojects/unicode-ident.wrap
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+# SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
+
+[wrap-file]
+directory = unicode-ident-1.0.12
+source_url = https://crates.io/api/v1/crates/unicode-ident/1.0.12/download
+source_filename = unicode-ident-1.0.12.tar.gz
+source_hash = 3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b
+patch_directory = unicode-ident
diff --git a/host/start-vm/tests/bridge_add-%d.c b/host/start-vmm/tests/bridge_add-%d.c
index 17e6013..17e6013 100644
--- a/host/start-vm/tests/bridge_add-%d.c
+++ b/host/start-vmm/tests/bridge_add-%d.c
diff --git a/host/start-vm/tests/bridge_add-name-too-long.c b/host/start-vmm/tests/bridge_add-name-too-long.c
index ec81373..ec81373 100644
--- a/host/start-vm/tests/bridge_add-name-too-long.c
+++ b/host/start-vmm/tests/bridge_add-name-too-long.c
diff --git a/host/start-vm/tests/bridge_add.c b/host/start-vmm/tests/bridge_add.c
index 693a11f..693a11f 100644
--- a/host/start-vm/tests/bridge_add.c
+++ b/host/start-vmm/tests/bridge_add.c
diff --git a/host/start-vm/tests/bridge_add_if.c b/host/start-vmm/tests/bridge_add_if.c
index f65151c..f65151c 100644
--- a/host/start-vm/tests/bridge_add_if.c
+++ b/host/start-vmm/tests/bridge_add_if.c
diff --git a/host/start-vm/tests/bridge_remove.c b/host/start-vmm/tests/bridge_remove.c
index 9de41fe..9de41fe 100644
--- a/host/start-vm/tests/bridge_remove.c
+++ b/host/start-vmm/tests/bridge_remove.c
diff --git a/host/start-vm/tests/bridge_remove_if.c b/host/start-vmm/tests/bridge_remove_if.c
index ebc7ce2..ebc7ce2 100644
--- a/host/start-vm/tests/bridge_remove_if.c
+++ b/host/start-vmm/tests/bridge_remove_if.c
diff --git a/host/start-vm/tests/helper.rs b/host/start-vmm/tests/helper.rs
index abaf973..9ce55f3 100644
--- a/host/start-vm/tests/helper.rs
+++ b/host/start-vmm/tests/helper.rs
@@ -9,7 +9,7 @@ use std::os::unix::prelude::*;
 use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 
-use start_vm::prog_name;
+use start_vmm::prog_name;
 
 extern "C" {
     fn mkdtemp(template: *mut c_char) -> *mut c_char;
@@ -31,7 +31,7 @@ impl TempDir {
         });
 
         let mut dirname = tmpdir.clone().into_os_string().into_vec();
-        dirname.extend_from_slice(b"/spectrum-start-vm-test-");
+        dirname.extend_from_slice(b"/spectrum-start-vmm-test-");
         dirname.extend_from_slice(&prog_name().into_bytes());
         dirname.extend_from_slice(b".XXXXXX\0");
 
diff --git a/host/start-vm/tests/if_down.c b/host/start-vmm/tests/if_down.c
index c912f4e..c912f4e 100644
--- a/host/start-vm/tests/if_down.c
+++ b/host/start-vmm/tests/if_down.c
diff --git a/host/start-vm/tests/if_rename-%d.c b/host/start-vmm/tests/if_rename-%d.c
index 68dbea2..68dbea2 100644
--- a/host/start-vm/tests/if_rename-%d.c
+++ b/host/start-vmm/tests/if_rename-%d.c
diff --git a/host/start-vm/tests/if_rename-name-too-long.c b/host/start-vmm/tests/if_rename-name-too-long.c
index 668824c..668824c 100644
--- a/host/start-vm/tests/if_rename-name-too-long.c
+++ b/host/start-vmm/tests/if_rename-name-too-long.c
diff --git a/host/start-vm/tests/if_rename.c b/host/start-vmm/tests/if_rename.c
index c73ae92..c73ae92 100644
--- a/host/start-vm/tests/if_rename.c
+++ b/host/start-vmm/tests/if_rename.c
diff --git a/host/start-vm/tests/if_up.c b/host/start-vmm/tests/if_up.c
index 33acb36..33acb36 100644
--- a/host/start-vm/tests/if_up.c
+++ b/host/start-vmm/tests/if_up.c
diff --git a/host/start-vm/tests/meson.build b/host/start-vmm/tests/meson.build
index 9d652ff..7f6bd08 100644
--- a/host/start-vm/tests/meson.build
+++ b/host/start-vmm/tests/meson.build
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: EUPL-1.2+
-# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+# SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
 
 rust_helper = static_library('test_helper', 'helper.rs',
-  link_with : [c_lib, rust_lib])
+  dependencies : rust_lib_dep)
 
 test('if_up', executable('if_up', 'if_up.c', '../net-util.c'))
 test('if_rename', executable('if_rename', 'if_rename.c', '../net-util.c'))
@@ -30,8 +30,14 @@ test('tap_open (name too long)', executable('tap_open-name-too-long',
   'tap_open-name-too-long.c', '../net-util.c'))
 
 test('vm_command-basic', executable('vm_command-basic',
-  'vm_command-basic.rs', link_with : [rust_lib, rust_helper]))
+  'vm_command-basic.rs',
+  dependencies : rust_lib_dep,
+  link_with : rust_helper))
 test('vm_command-multiple-disks', executable('vm_command-multiple-disks',
-  'vm_command-multiple-disks.rs', link_with : [rust_lib, rust_helper]))
+  'vm_command-multiple-disks.rs',
+  dependencies : rust_lib_dep,
+  link_with : rust_helper))
 test('vm_command-shared-dir', executable('vm_command-shared-dir',
-  'vm_command-shared-dir.rs', link_with : [rust_lib, rust_helper]))
+  'vm_command-shared-dir.rs',
+  dependencies : rust_lib_dep,
+  link_with : rust_helper))
diff --git a/host/start-vm/tests/tap_open-name-too-long.c b/host/start-vmm/tests/tap_open-name-too-long.c
index ba4ebd6..ba4ebd6 100644
--- a/host/start-vm/tests/tap_open-name-too-long.c
+++ b/host/start-vmm/tests/tap_open-name-too-long.c
diff --git a/host/start-vm/tests/tap_open.c b/host/start-vmm/tests/tap_open.c
index bf5d00c..bf5d00c 100644
--- a/host/start-vm/tests/tap_open.c
+++ b/host/start-vmm/tests/tap_open.c
diff --git a/host/start-vmm/tests/vm_command-basic.rs b/host/start-vmm/tests/vm_command-basic.rs
new file mode 100644
index 0000000..305e98c
--- /dev/null
+++ b/host/start-vmm/tests/vm_command-basic.rs
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+use std::fs::{create_dir_all, File};
+use std::path::PathBuf;
+
+use start_vmm::vm_config;
+use test_helper::TempDir;
+
+fn main() -> std::io::Result<()> {
+    let tmp_dir = TempDir::new()?;
+
+    let kernel_path = tmp_dir.path().join("testvm/config/vmlinux");
+    let image_path = tmp_dir.path().join("testvm/config/blk/root.img");
+
+    create_dir_all(kernel_path.parent().unwrap())?;
+    create_dir_all(image_path.parent().unwrap())?;
+    File::create(&kernel_path)?;
+    File::create(&image_path)?;
+
+    let mut config = vm_config("testvm", tmp_dir.path()).unwrap();
+
+    assert_eq!(config.console.mode, "Pty");
+    assert_eq!(config.disks.len(), 1);
+    let disk1 = config.disks.pop().unwrap();
+    assert_eq!(PathBuf::from(disk1.path), image_path);
+    assert!(disk1.readonly);
+    assert_eq!(PathBuf::from(config.payload.kernel), kernel_path);
+    assert_eq!(config.payload.cmdline, "console=ttyS0 root=PARTLABEL=root");
+    assert_eq!(config.memory.size, 0x10000000);
+    assert!(config.memory.shared);
+    assert_eq!(config.serial.mode, "File");
+    assert_eq!(config.serial.file.unwrap(), "/run/testvm.log");
+
+    Ok(())
+}
diff --git a/host/start-vmm/tests/vm_command-multiple-disks.rs b/host/start-vmm/tests/vm_command-multiple-disks.rs
new file mode 100644
index 0000000..3227a91
--- /dev/null
+++ b/host/start-vmm/tests/vm_command-multiple-disks.rs
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+use std::collections::BTreeSet;
+use std::fs::{create_dir, create_dir_all, File};
+use std::os::unix::fs::symlink;
+use std::path::PathBuf;
+
+use start_vmm::vm_config;
+use test_helper::TempDir;
+
+fn main() -> std::io::Result<()> {
+    let tmp_dir = TempDir::new()?;
+
+    let vm_config_dir = tmp_dir.path().join("testvm/config");
+
+    create_dir_all(&vm_config_dir)?;
+    File::create(vm_config_dir.join("vmlinux"))?;
+    create_dir(vm_config_dir.join("blk"))?;
+
+    let image_paths: BTreeSet<_> = (1..=2)
+        .map(|n| vm_config_dir.join(format!("blk/disk{n}.img")))
+        .collect();
+
+    for image_path in &image_paths {
+        symlink("/dev/null", image_path)?;
+    }
+
+    let config = vm_config("testvm", tmp_dir.path()).unwrap();
+    assert_eq!(config.disks.len(), 2);
+    assert!(config.disks.iter().all(|disk| disk.readonly));
+
+    let actual_paths: BTreeSet<_> = config
+        .disks
+        .into_iter()
+        .map(|disk| PathBuf::from(disk.path))
+        .collect();
+
+    assert_eq!(actual_paths, image_paths);
+
+    Ok(())
+}
diff --git a/host/start-vmm/tests/vm_command-shared-dir.rs b/host/start-vmm/tests/vm_command-shared-dir.rs
new file mode 100644
index 0000000..b00362e
--- /dev/null
+++ b/host/start-vmm/tests/vm_command-shared-dir.rs
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022-2023 Alyssa Ross <hi@alyssa.is>
+
+use std::collections::BTreeSet;
+use std::fs::{create_dir, create_dir_all, File};
+use std::os::unix::fs::symlink;
+
+use start_vmm::vm_config;
+use test_helper::TempDir;
+
+fn main() -> std::io::Result<()> {
+    let tmp_dir = TempDir::new()?;
+
+    let vm_config_dir = tmp_dir.path().join("testvm/config");
+
+    create_dir_all(&vm_config_dir)?;
+    File::create(vm_config_dir.join("vmlinux"))?;
+    create_dir(vm_config_dir.join("blk"))?;
+    symlink("/dev/null", vm_config_dir.join("blk/root.img"))?;
+
+    create_dir(vm_config_dir.join("shared-dirs"))?;
+
+    create_dir(vm_config_dir.join("shared-dirs/dir1"))?;
+    symlink("/", vm_config_dir.join("shared-dirs/dir1/dir"))?;
+
+    create_dir(vm_config_dir.join("shared-dirs/dir2"))?;
+    symlink("/", vm_config_dir.join("shared-dirs/dir2/dir"))?;
+
+    let config = vm_config("testvm", tmp_dir.path()).unwrap();
+    assert_eq!(config.fs.len(), 2);
+
+    let mut actual_tags = BTreeSet::new();
+    let mut actual_sockets = BTreeSet::new();
+
+    for fs in config.fs {
+        actual_tags.insert(fs.tag);
+        actual_sockets.insert(fs.socket);
+    }
+
+    let expected_tags = (1..=2).map(|i| format!("dir{i}")).collect();
+    assert_eq!(actual_tags, expected_tags);
+
+    let expected_sockets = (1..=2)
+        .map(|i| format!("../fs-testvm-dir{i}/env/virtiofsd.sock"))
+        .collect();
+    assert_eq!(actual_sockets, expected_sockets);
+
+    Ok(())
+}
diff --git a/host/start-vm/unix.c b/host/start-vmm/unix.c
index 43143fd..43143fd 100644
--- a/host/start-vm/unix.c
+++ b/host/start-vmm/unix.c
diff --git a/host/start-vm/unix.rs b/host/start-vmm/unix.rs
index 8213497..8213497 100644
--- a/host/start-vm/unix.rs
+++ b/host/start-vmm/unix.rs
diff --git a/pkgs/default.nix b/pkgs/default.nix
index ed2762d..ca072f5 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -7,7 +7,15 @@ let
   config = import ../lib/config.nix args;
   pkgs = import ./overlaid.nix ({ elaboratedConfig = config; } // args);
 
-  inherit (pkgs.lib) cleanSource fileset makeScope optionalAttrs;
+  inherit (pkgs.lib) cleanSource fileset makeScope optionalAttrs sourceByRegex;
+
+  subprojects =
+    project:
+    let dir = project + "/subprojects"; in
+    fileset.difference dir (fileset.fromSource (sourceByRegex dir [
+      ".*\.wrap"
+      "packagefiles(/.*)?"
+    ]));
 
   makeScopeWithSplicing = pkgs: pkgs.makeScopeWithSplicing' {
     otherSplices = {
@@ -29,7 +37,7 @@ let
 
     lseek = self.callSpectrumPackage ../tools/lseek {};
     rootfs = self.callSpectrumPackage ../host/rootfs {};
-    start-vm = self.callSpectrumPackage ../host/start-vm {};
+    start-vmm = self.callSpectrumPackage ../host/start-vmm {};
     run-spectrum-vm = self.callSpectrumPackage ../scripts/run-spectrum-vm.nix {};
 
     # Packages from the overlay, so it's possible to build them from
@@ -40,7 +48,9 @@ let
 
     srcWithNix = fileset.difference
       (fileset.fromSource (cleanSource ../.))
-      (fileset.unions (map fileset.maybeMissing [
+      (fileset.unions ([
+        (subprojects ../host/start-vmm)
+      ] ++ map fileset.maybeMissing [
         ../Documentation/.jekyll-cache
         ../Documentation/_site
         ../Documentation/diagrams/stack.svg
diff --git a/release/checks/pkg-tests.nix b/release/checks/pkg-tests.nix
index f51ba42..7a41e8d 100644
--- a/release/checks/pkg-tests.nix
+++ b/release/checks/pkg-tests.nix
@@ -2,14 +2,14 @@
 # SPDX-FileCopyrightText: 2023 Alyssa Ross <hi@alyssa.is>
 
 import ../../lib/call-package.nix (
-{ callSpectrumPackage, lseek, start-vm, lib }:
+{ callSpectrumPackage, lseek, start-vmm, lib }:
 
 {
   recurseForDerivations = true;
 
   lseek = lib.recurseIntoAttrs lseek.tests;
 
-  start-vm = lib.recurseIntoAttrs start-vm.tests;
+  start-vmm = lib.recurseIntoAttrs start-vmm.tests;
 
   run-spectrum-vm = lib.recurseIntoAttrs
     (callSpectrumPackage ../../scripts/run-spectrum-vm.nix {}).tests;
diff --git a/scripts/run-spectrum-vm.c b/scripts/run-spectrum-vm.c
index 8e0978a..1c1cb1d 100644
--- a/scripts/run-spectrum-vm.c
+++ b/scripts/run-spectrum-vm.c
@@ -161,8 +161,8 @@ int main(void)
 	if (fd != 3)
 		close(fd);
 
-	if ((fd = open(START_VM_PATH, O_PATH|O_CLOEXEC)) == -1)
-		err(EXIT_FAILURE, "open " START_VM_PATH);
+	if ((fd = open(START_VMM_PATH, O_PATH|O_CLOEXEC)) == -1)
+		err(EXIT_FAILURE, "open " START_VMM_PATH);
 
 	if (unshare(CLONE_NEWUSER|CLONE_NEWNS) == -1)
 		err(EXIT_FAILURE, "unshare");
@@ -177,6 +177,6 @@ int main(void)
 	if (chroot(dir_path) == -1)
 		err(EXIT_FAILURE, "chroot");
 
-	fexecve(fd, (char *const []){"start-vm", NULL}, (char *const []){"PATH=/bin", NULL});
-	err(EXIT_FAILURE, "exec " START_VM_PATH);
+	fexecve(fd, (char *const []){"start-vmm", NULL}, (char *const []){"PATH=/bin", NULL});
+	err(EXIT_FAILURE, "exec " START_VMM_PATH);
 }
diff --git a/scripts/run-spectrum-vm.nix b/scripts/run-spectrum-vm.nix
index a6b40bb..0c18d02 100644
--- a/scripts/run-spectrum-vm.nix
+++ b/scripts/run-spectrum-vm.nix
@@ -4,7 +4,7 @@
 { run ? ../vm/app/poweroff.nix, ... } @ args:
 
 import ../lib/call-package.nix (
-{ callSpectrumPackage, start-vm
+{ callSpectrumPackage, start-vmm
 , lib, runCommand, runCommandCC, clang-tools, cloud-hypervisor, virtiofsd
 }:
 
@@ -14,7 +14,7 @@ let
     ''-DAPPVM_PATH="${callSpectrumPackage ../img/app {}}"''
     ''-DCONFIG_PATH="${callSpectrumPackage run {}}"''
     ''-DCLOUD_HYPERVISOR_PATH="${lib.getExe cloud-hypervisor}"''
-    ''-DSTART_VM_PATH="${lib.getExe start-vm}"''
+    ''-DSTART_VMM_PATH="${lib.getExe start-vmm}"''
     ''-DVIRTIOFSD_PATH="${lib.getExe virtiofsd}"''
   ];
 in