host/start-vm: support multiple block devices

This is the first step to being able to do things like having a static
base image for an application VM, and another image containing the
application to run.

When we actually use multiple images we'll need to change how we boot
so we know the kernel is booting from the right image — probably by
PARTUUID or PARTLABEL — because we can't guarantee how the kernel
orders disks inside the VM.

Signed-off-by: Alyssa Ross <hi@alyssa.is>
Message-Id: <20221009114036.463071-2-hi@alyssa.is>

6 files changed, 55 insertions, 29 deletions

diff --git a/Documentation/getting-started/creating-vms.adoc b/Documentation/getting-started/creating-vms.adoc
index 725316c..a2ad952 100644
--- a/Documentation/getting-started/creating-vms.adoc
+++ b/Documentation/getting-started/creating-vms.adoc
@@ -16,8 +16,9 @@ The directory can contain the following files:
 vmlinux:: An uncompressed Linux kernel image for the VM to boot.
 *Required.*
 
-rootfs.ext4:: A disk image that will be provided to the guest as a
-virtio-blk device.  *Required.*
+blk:: A directory containing disk images (with file names ending in
+".img") that will be provided to the guest as a virtio-blk device.
+Order is not guaranteed.  At least one image is *required*.
 
 providers/net:: A directory containing a file named for each VM that
 should provide networking to this VM.  The contents of these files are
@@ -34,7 +35,8 @@ appvm-lynx
 ├── providers/
 │   └── net/
 │       └── netvm
-├── rootfs.ext4
+├── blk/
+│   └── root.img
 └── vmlinux*
 ----
 
diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs
index 1230a6e..5d43a3e 100644
--- a/host/start-vm/lib.rs
+++ b/host/start-vm/lib.rs
@@ -81,12 +81,36 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
         Err(e) => return Err(format!("reading directory {:?}: {}", net_providers_dir, e)),
     }
 
-    command.arg("--disk").arg({
-        let mut disk = OsString::from("path=/ext/svc/data/");
-        disk.push(&vm_name);
-        disk.push("/rootfs.ext4,readonly=on");
-        disk
-    });
+    command.arg("--disk");
+
+    let blk_dir = config_dir.join("blk");
+    match blk_dir.read_dir() {
+        Ok(entries) => {
+            for result in entries {
+                let entry = result
+                    .map_err(|e| format!("examining directory entry: {}", e))?
+                    .path();
+
+                if entry.extension() != Some(OsStr::new("img")) {
+                    continue;
+                }
+
+                if entry.as_os_str().as_bytes().contains(&b',') {
+                    return Err(format!("illegal ',' character in path {:?}", entry));
+                }
+
+                let mut arg = OsString::from("path=");
+                arg.push(entry);
+                arg.push(",readonly=on");
+                command.arg(arg);
+            }
+        }
+        Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)),
+    }
+
+    if command.get_args().last() == Some(OsStr::new("--disk")) {
+        return Err("no block devices specified".to_string());
+    }
 
     command.arg("--serial").arg({
         let mut serial = OsString::from("file=/run/");
diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs
index b2edb7c..a577a71 100644
--- a/host/start-vm/tests/vm_command-basic.rs
+++ b/host/start-vm/tests/vm_command-basic.rs
@@ -14,7 +14,7 @@ fn main() -> std::io::Result<()> {
     create_dir(&service_dir)?;
 
     let kernel_path = tmp_dir.path().join("svc/data/testvm/vmlinux");
-    let image_path = tmp_dir.path().join("svc/data/testvm/rootfs.ext4");
+    let image_path = tmp_dir.path().join("svc/data/testvm/blk/root.img");
 
     create_dir_all(kernel_path.parent().unwrap())?;
     create_dir_all(image_path.parent().unwrap())?;
@@ -43,7 +43,7 @@ fn main() -> std::io::Result<()> {
         OsStr::new("--kernel"),
         kernel_path.as_os_str(),
         OsStr::new("--disk"),
-        OsStr::new("path=/ext/svc/data/testvm/rootfs.ext4,readonly=on"),
+        &expected_disk_arg,
         OsStr::new("--serial"),
         OsStr::new("file=/run/testvm.log"),
     ];
diff --git a/vm/app/catgirl/Makefile b/vm/app/catgirl/Makefile
index a32826b..9016745 100644
--- a/vm/app/catgirl/Makefile
+++ b/vm/app/catgirl/Makefile
@@ -12,7 +12,7 @@ VMM = qemu
 HOST_FILES = host/data/appvm-catgirl/providers/net/netvm
 
 HOST_BUILD_FILES = \
-	build/host/data/appvm-catgirl/rootfs.ext4 \
+	build/host/data/appvm-catgirl/blk/root.img \
 	build/host/data/appvm-catgirl/vmlinux
 
 # We produce a directory, but that doesn't play nice with Make,
@@ -35,7 +35,7 @@ build/host/data/appvm-catgirl/vmlinux: $(VMLINUX)
 
 # tar2ext4 will leave half a filesystem behind if it's interrupted
 # half way through.
-build/host/data/appvm-catgirl/rootfs.ext4: build/rootfs.tar
+build/host/data/appvm-catgirl/blk/root.img: build/rootfs.tar
 	mkdir -p $$(dirname $@)
 	tar2ext4 -i build/rootfs.tar -o $@.tmp
 	mv $@.tmp $@
@@ -92,9 +92,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES)
 	    s6-rc-compile $@ $$dir; \
 	    exit=$$?; rm -r $$dir; exit $$exit
 
-run-qemu: build/host/data/appvm-catgirl/rootfs.ext4
+run-qemu: build/host/data/appvm-catgirl/blk/root.img
 	$(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \
-	  -drive file=build/host/data/appvm-catgirl/rootfs.ext4,if=virtio,format=raw,readonly=on \
+	  -drive file=build/host/data/appvm-catgirl/blk/root.img,if=virtio,format=raw,readonly=on \
 	  -append "console=ttyS0 root=/dev/vda" \
 	  -netdev user,id=net0 \
 	  -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \
@@ -103,11 +103,11 @@ run-qemu: build/host/data/appvm-catgirl/rootfs.ext4
 	  -device virtconsole,chardev=virtiocon0
 .PHONY: run-qemu
 
-run-cloud-hypervisor: build/host/data/appvm-catgirl/rootfs.ext4
+run-cloud-hypervisor: build/host/data/appvm-catgirl/blk/root.img
 	$(CLOUD_HYPERVISOR) \
 	    --api-socket path=vmm.sock \
 	    --memory size=128M \
-	    --disk path=build/host/data/appvm-catgirl/rootfs.ext4,readonly=on \
+	    --disk path=build/host/data/appvm-catgirl/blk/root.img,readonly=on \
 	    --net tap=tap0,mac=0A:B3:EC:00:00:00 \
 	    --kernel $(KERNEL) \
 	    --cmdline "console=ttyS0 root=/dev/vda" \
diff --git a/vm/app/lynx/Makefile b/vm/app/lynx/Makefile
index c25c9ab..ff87cb8 100644
--- a/vm/app/lynx/Makefile
+++ b/vm/app/lynx/Makefile
@@ -12,7 +12,7 @@ VMM = qemu
 HOST_FILES = host/data/appvm-lynx/providers/net/netvm
 
 HOST_BUILD_FILES = \
-	build/host/data/appvm-lynx/rootfs.ext4 \
+	build/host/data/appvm-lynx/blk/root.img \
 	build/host/data/appvm-lynx/vmlinux
 
 # We produce a directory, but that doesn't play nice with Make,
@@ -35,7 +35,7 @@ build/host/data/appvm-lynx/vmlinux: $(VMLINUX)
 
 # tar2ext4 will leave half a filesystem behind if it's interrupted
 # half way through.
-build/host/data/appvm-lynx/rootfs.ext4: build/rootfs.tar
+build/host/data/appvm-lynx/blk/root.img: build/rootfs.tar
 	mkdir -p $$(dirname $@)
 	tar2ext4 -i build/rootfs.tar -o $@.tmp
 	mv $@.tmp $@
@@ -91,9 +91,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES)
 	    s6-rc-compile $@ $$dir; \
 	    exit=$$?; rm -r $$dir; exit $$exit
 
-run-qemu: build/host/data/appvm-lynx/rootfs.ext4
+run-qemu: build/host/data/appvm-lynx/blk/root.img
 	$(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \
-	  -drive file=build/host/data/appvm-lynx/rootfs.ext4,if=virtio,format=raw,readonly=on \
+	  -drive file=build/host/data/appvm-lynx/blk/root.img,if=virtio,format=raw,readonly=on \
 	  -append "console=ttyS0 root=/dev/vda" \
 	  -netdev user,id=net0 \
 	  -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \
@@ -102,11 +102,11 @@ run-qemu: build/host/data/appvm-lynx/rootfs.ext4
 	  -device virtconsole,chardev=virtiocon0
 .PHONY: run-qemu
 
-run-cloud-hypervisor: build/host/data/appvm-lynx/rootfs.ext4
+run-cloud-hypervisor: build/host/data/appvm-lynx/blk/root.img
 	$(CLOUD_HYPERVISOR) \
 	    --api-socket path=vmm.sock \
 	    --memory size=128M \
-	    --disk path=build/host/data/appvm-lynx/rootfs.ext4,readonly=on \
+	    --disk path=build/host/data/appvm-lynx/blk/root.img,readonly=on \
 	    --net tap=tap0,mac=0A:B3:EC:00:00:00 \
 	    --kernel $(KERNEL) \
 	    --cmdline "console=ttyS0 root=/dev/vda" \
diff --git a/vm/sys/net/Makefile b/vm/sys/net/Makefile
index 7cb7a5f..0b5fe8f 100644
--- a/vm/sys/net/Makefile
+++ b/vm/sys/net/Makefile
@@ -10,7 +10,7 @@ CLOUD_HYPERVISOR = cloud-hypervisor
 VMM = qemu
 
 HOST_BUILD_FILES = \
-	build/host/data/netvm/rootfs.ext4 \
+	build/host/data/netvm/blk/root.img \
 	build/host/data/netvm/vmlinux
 
 # We produce a directory, but that doesn't play nice with Make,
@@ -32,7 +32,7 @@ build/host/data/netvm/vmlinux: $(VMLINUX)
 
 # tar2ext4 will leave half a filesystem behind if it's interrupted
 # half way through.
-build/host/data/netvm/rootfs.ext4: build/rootfs.tar
+build/host/data/netvm/blk/root.img: build/rootfs.tar
 	mkdir -p $$(dirname $@)
 	tar2ext4 -i build/rootfs.tar -o $@.tmp
 	mv $@.tmp $@
@@ -97,9 +97,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES)
 	    s6-rc-compile $@ $$dir; \
 	    exit=$$?; rm -r $$dir; exit $$exit
 
-run-qemu: build/host/data/netvm/rootfs.ext4
+run-qemu: build/host/data/netvm/blk/root.img
 	$(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \
-	  -drive file=build/host/data/netvm/rootfs.ext4,if=virtio,format=raw,readonly=on \
+	  -drive file=build/host/data/netvm/blk/root.img,if=virtio,format=raw,readonly=on \
 	  -append "console=ttyS0 root=/dev/vda" \
 	  -netdev user,id=net0 \
 	  -device e1000e,netdev=net0 \
@@ -110,11 +110,11 @@ run-qemu: build/host/data/netvm/rootfs.ext4
 	  -device virtconsole,chardev=virtiocon0
 .PHONY: run-qemu
 
-run-cloud-hypervisor: build/host/data/netvm/rootfs.ext4
+run-cloud-hypervisor: build/host/data/netvm/blk/root.img
 	$(CLOUD_HYPERVISOR) \
 	    --api-socket path=vmm.sock \
 	    --memory size=128M \
-	    --disk path=build/host/data/netvm/rootfs.ext4,readonly=on \
+	    --disk path=build/host/data/netvm/blk/root.img,readonly=on \
 	    --net tap=tap0 tap=tap1,mac=0A:B3:EC:80:00:00 \
 	    --kernel $(KERNEL) \
 	    --cmdline "console=ttyS0 root=/dev/vda" \