Arch: Big refactor and add an empty PCI bus

When setting up IO, accept an optional PciRoot device to put on the IO
bus.

For aarch64, it's currently ignored. For x86_64, it will be added at
0xcf8.

 break up mmio device creation and registration

Moving forward registration will be handled by the architecture specific
code. However, creation will be handled by the common code. To make that
easier split up the two steps so a list of devices is created, then each
is registered later.

Start moving to a model where the configuration generates a set of
components that are passed to the architecture. The architecture will
crate a VM from the components.

Break up the big run_config function and move architecture specific
parts to the various architectures.

This doesn't refactor the function calls each architecture makes, but
moves the setup flow in to the arch impls so that they can diverge in
the future.

Change-Id: I5b10d092896606796dc0c9afc5e34a1b288b867b
Signed-off-by: Dylan Reid <dgreid@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/1099860
Commit-Ready: Daniel Verkamp <dverkamp@chromium.org>
Tested-by: Daniel Verkamp <dverkamp@chromium.org>

9 files changed, 542 insertions, 483 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 4eb57b9..8ac7e4f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -29,6 +29,7 @@ name = "arch"
 version = "0.1.0"
 dependencies = [
  "devices 0.1.0",
+ "io_jail 0.1.0",
  "kernel_cmdline 0.1.0",
  "kvm 0.1.0",
  "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs
index 86fa03a..cb39897 100644
--- a/aarch64/src/lib.rs
+++ b/aarch64/src/lib.rs
@@ -14,16 +14,18 @@ extern crate sys_util;
 extern crate resources;
 
 use std::error::{self, Error as Aarch64Error};
+use std::ffi::{CStr, CString};
 use std::fmt::{self, Display};
 use std::fs::File;
-use std::io::stdout;
+use std::io::{self, stdout};
 use std::sync::{Arc, Mutex};
-use std::ffi::CStr;
+use std::os::unix::io::FromRawFd;
+use std::os::unix::net::UnixDatagram;
 
+use arch::{RunnableLinuxVm, VirtioDeviceStub, VmComponents};
 use devices::{Bus, PciInterruptPin};
 use sys_util::{EventFd, GuestAddress, GuestMemory};
 use resources::{AddressRanges, SystemAllocator};
-use std::os::unix::io::FromRawFd;
 
 use kvm::*;
 use kvm_sys::kvm_device_attr;
@@ -105,12 +107,30 @@ const AARCH64_IRQ_BASE: u32 = 1;
 
 #[derive(Debug)]
 pub enum Error {
+    /// Error Adding a PCI device.
+    AddPciDev(devices::PciRootError),
+    /// Unable to clone an EventFd
+    CloneEventFd(sys_util::Error),
+    /// Error creating kernel command line.
+    Cmdline(kernel_cmdline::Error),
+    /// Unable to make an EventFd
+    CreateEventFd(sys_util::Error),
+    /// Unable to create Kvm.
+    CreateKvm(sys_util::Error),
+    /// Unable to create a PciRoot hub.
+    CreatePciRoot(devices::PciRootError),
+    /// Unable to create socket.
+    CreateSocket(io::Error),
+    /// Unable to create Vcpu.
+    CreateVcpu(sys_util::Error),
     /// FDT could not be created
     FDTCreateFailure(Box<error::Error>),
     /// Kernel could not be loaded
     KernelLoadFailure,
     /// Failure to Create GIC
     CreateGICFailure(sys_util::Error),
+    /// Couldn't register virtio socket.
+    RegisterVsock(arch::MmioRegisterError),
     /// VCPU Init failed
     VCPUInitFailure,
     /// VCPU Set one reg failed
@@ -120,12 +140,21 @@ pub enum Error {
 impl error::Error for Error {
     fn description(&self) -> &str {
         match self {
+            &Error::AddPciDev(_) => "Failed to add device to PCI",
+            &Error::CloneEventFd(_) => "Unable to clone an EventFd",
+            &Error::Cmdline(_) => "the given kernel command line was invalid",
+            &Error::CreateEventFd(_) => "Unable to make an EventFd",
+            &Error::CreateKvm(_) => "failed to open /dev/kvm",
+            &Error::CreatePciRoot(_) => "failed to create a PCI root hub",
+            &Error::CreateSocket(_) => "failed to create socket",
+            &Error::CreateVcpu(_) => "failed to create VCPU",
             &Error::FDTCreateFailure(_) =>
                 "FDT could not be created",
             &Error::KernelLoadFailure =>
                 "Kernel cound not be loaded",
             &Error::CreateGICFailure(_) =>
                 "Failure to create GIC",
+            &Error::RegisterVsock(_) => "error registering virtual socket device",
             &Error::VCPUInitFailure =>
                 "Failed to initialize VCPU",
             &Error::VCPUSetRegFailure =>
@@ -156,6 +185,73 @@ fn fdt_offset(mem_size: u64) -> u64 {
 pub struct AArch64;
 
 impl arch::LinuxArch for AArch64 {
+    fn build_vm<F>(mut components: VmComponents, virtio_devs: F) -> Result<RunnableLinuxVm>
+        where
+            F: FnOnce(&GuestMemory, &EventFd) -> Result<Vec<VirtioDeviceStub>>
+    {
+        let mut resources = Self::get_resource_allocator(components.memory_mb,
+                                                         components.wayland_dmabuf);
+        let mem = Self::setup_memory(components.memory_mb)?;
+        let kvm = Kvm::new().map_err(Error::CreateKvm)?;
+        let mut vm = Self::create_vm(&kvm, mem.clone())?;
+
+        let vcpu_count = components.vcpu_count;
+        let mut vcpus = Vec::with_capacity(vcpu_count as usize);
+        for cpu_id in 0..vcpu_count {
+            let vcpu = Vcpu::new(cpu_id as libc::c_ulong, &kvm, &vm)
+                .map_err(Error::CreateVcpu)?;
+            Self::configure_vcpu(vm.get_memory(), &kvm, &vm, &vcpu,
+                                 cpu_id as u64, vcpu_count as u64)?;
+            vcpus.push(vcpu);
+        }
+
+        let irq_chip = Self::create_irq_chip(&vm)?;
+        let mut cmdline = Self::get_base_linux_cmdline();
+
+        let mut mmio_bus = devices::Bus::new();
+
+        let (pci, pci_irqs) = components.pci_devices.generate_root(&mut mmio_bus, &mut resources)
+            .map_err(Error::CreatePciRoot)?;
+
+        let exit_evt = EventFd::new().map_err(Error::CreateEventFd)?;
+        let (io_bus, stdio_serial) = Self::setup_io_bus()?;
+
+        // Create a list of mmio devices to be added.
+        let mmio_devs = virtio_devs(&mem, &exit_evt)?;
+
+        Self::add_arch_devs(&mut vm, &mut mmio_bus)?;
+
+        for stub in mmio_devs {
+            arch::register_mmio(&mut mmio_bus, &mut vm, stub.dev, stub.jail,
+                                &mut resources, &mut cmdline)
+                .map_err(Error::RegisterVsock)?;
+        }
+
+        for param in components.extra_kernel_params {
+            cmdline.insert_str(&param).map_err(Error::Cmdline)?;
+        }
+
+        // separate out load_kernel from other setup to get a specific error for
+        // kernel loading
+        Self::load_kernel(&mem, &mut components.kernel_image)?;
+        Self::setup_system_memory(&mem, components.memory_mb, vcpu_count,
+                                  &CString::new(cmdline).unwrap(), pci_irqs)?;
+
+        Ok(RunnableLinuxVm {
+            vm,
+            kvm,
+            resources,
+            stdio_serial,
+            exit_evt,
+            vcpus,
+            irq_chip,
+            io_bus,
+            mmio_bus,
+        })
+    }
+}
+
+impl AArch64 {
     /// Loads the kernel from an open file.
     ///
     /// # Arguments
@@ -334,7 +430,7 @@ impl arch::LinuxArch for AArch64 {
         Ok(Some(vgic_fd))
     }
 
-    fn setup_io_bus(_vm: &mut Vm, _exit_evt: EventFd)
+    fn setup_io_bus()
                     -> Result<(devices::Bus, Arc<Mutex<devices::Serial>>)> {
         // ARM doesn't really use the io bus like x86, instead we have a
         // separate serial device that is returned as a separate object.
@@ -395,5 +491,4 @@ impl arch::LinuxArch for AArch64 {
         }
         Ok(())
     }
-
 }
diff --git a/arch/Cargo.toml b/arch/Cargo.toml
index 20c5466..4a22f17 100644
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -5,6 +5,7 @@ authors = ["The Chromium OS Authors"]
 
 [dependencies]
 devices = { path = "../devices" }
+io_jail = { path = "../io_jail" }
 kvm = { path = "../kvm" }
 sys_util = { path = "../sys_util" }
 resources = { path = "../resources" }
diff --git a/arch/src/lib.rs b/arch/src/lib.rs
index e2d447c..d7a1a63 100644
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+extern crate io_jail;
 extern crate sys_util;
 extern crate resources;
 extern crate kernel_cmdline;
@@ -9,114 +10,154 @@ extern crate kvm;
 extern crate libc;
 extern crate devices;
 
-use std::ffi::CStr;
+use std::fmt;
 use std::fs::File;
+use std::os::unix::io::{AsRawFd, RawFd};
 use std::result;
 use std::sync::{Arc, Mutex};
 
-use devices::Bus;
-use kvm::{Kvm, Vm, Vcpu};
-use sys_util::{EventFd, GuestMemory};
+use devices::{Bus, PciDeviceList, Serial};
+use devices::virtio::VirtioDevice;
+use io_jail::Minijail;
+use kvm::{IoeventAddress, Kvm, Vm, Vcpu};
+use sys_util::{EventFd, GuestMemory, syslog};
 use resources::SystemAllocator;
 
 pub type Result<T> = result::Result<T, Box<std::error::Error>>;
 
+/// Holds the pieces needed to build a VM. Passed to `build_vm` in the `LinuxArch` trait below to
+/// create a `RunnableLinuxVm`.
+pub struct VmComponents {
+    pub pci_devices: PciDeviceList,
+    pub memory_mb: u64,
+    pub vcpu_count: u32,
+    pub kernel_image: File,
+    pub extra_kernel_params: Vec<String>,
+    pub wayland_dmabuf: bool,
+}
+
+/// Holds the elements needed to run a Linux VM. Created by `build_vm`.
+pub struct RunnableLinuxVm {
+    pub vm: Vm,
+    pub kvm: Kvm,
+    pub resources: SystemAllocator,
+    pub stdio_serial: Arc<Mutex<Serial>>,
+    pub exit_evt: EventFd,
+    pub vcpus: Vec<Vcpu>,
+    pub irq_chip: Option<File>,
+    pub io_bus: Bus,
+    pub mmio_bus: Bus,
+}
+
+/// The device and optional jail.
+pub struct VirtioDeviceStub {
+    pub dev: Box<VirtioDevice>,
+    pub jail: Option<Minijail>,
+}
+
 /// Trait which is implemented for each Linux Architecture in order to
 /// set up the memory, cpus, and system devices and to boot the kernel.
 pub trait LinuxArch {
-    /// Loads the kernel from an open file.
+    /// Takes `VmComponents` and generates a `RunnableLinuxVm`.
     ///
     /// # Arguments
     ///
-    /// * `mem` - The memory to be used by the guest.
-    /// * `kernel_image` - the File object for the specified kernel.
-    fn load_kernel(mem: &GuestMemory, kernel_image: &mut File) -> Result<()>;
+    /// * `components` - Parts to use to build the VM.
+    /// * `virtio_devs` - Function to generate a list of virtio devices.
+    fn build_vm<F>(components: VmComponents, virtio_devs: F) -> Result<RunnableLinuxVm>
+        where
+            F: FnOnce(&GuestMemory, &EventFd) -> Result<Vec<VirtioDeviceStub>>;
+}
 
-    /// Configures the system memory space should be called once per vm before
-    /// starting vcpu threads.
-    ///
-    /// # Arguments
-    ///
-    /// * `mem` - The memory to be used by the guest
-    /// * `mem_size` - The size in bytes of system memory
-    /// * `vcpu_count` - Number of virtual CPUs the guest will have
-    /// * `cmdline` - the kernel commandline
-    /// * `pci_irqs` - Any PCI irqs that need to be configured (Interrupt Line, PCI pin).
-    fn setup_system_memory(mem: &GuestMemory,
-                           mem_size: u64,
-                           vcpu_count: u32,
-                           cmdline: &CStr,
-                           pci_irqs: Vec<(u32, devices::PciInterruptPin)>) -> Result<()>;
-
-    /// Creates a new VM object and initializes architecture specific devices
-    ///
-    /// # Arguments
-    ///
-    /// * `kvm` - The opened /dev/kvm object.
-    /// * `mem` - The memory to be used by the guest.
-    fn create_vm(kvm: &Kvm, mem: GuestMemory) -> Result<Vm>;
+/// Errors for device manager.
+#[derive(Debug)]
+pub enum MmioRegisterError {
+    /// Could not create the mmio device to wrap a VirtioDevice.
+    CreateMmioDevice(sys_util::Error),
+    /// Failed to register ioevent with VM.
+    RegisterIoevent(sys_util::Error),
+    /// Failed to register irq eventfd with VM.
+    RegisterIrqfd(sys_util::Error),
+    /// Failed to initialize proxy device for jailed device.
+    ProxyDeviceCreation(devices::ProxyError),
+    /// Appending to kernel command line failed.
+    Cmdline(kernel_cmdline::Error),
+    /// No more IRQs are available.
+    IrqsExhausted,
+    /// No more MMIO space available.
+    AddrsExhausted,
+}
 
-    /// This adds any early platform devices for this architecture.
-    ///
-    /// # Arguments
-    ///
-    /// * `vm` - The vm to add irqs to.
-    /// * `bus` - The bus to add devices to.
-    fn add_arch_devs(_vm: &mut Vm, _bus: &mut Bus) -> Result<()> { Ok(()) }
+impl fmt::Display for MmioRegisterError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            &MmioRegisterError::CreateMmioDevice(ref e) => write!(f, "failed to create mmio device: {:?}", e),
+            &MmioRegisterError::Cmdline(ref e) => {
+                write!(f, "unable to add device to kernel command line: {}", e)
+            }
+            &MmioRegisterError::RegisterIoevent(ref e) => {
+                write!(f, "failed to register ioevent to VM: {:?}", e)
+            }
+            &MmioRegisterError::RegisterIrqfd(ref e) => {
+                write!(f, "failed to register irq eventfd to VM: {:?}", e)
+            }
+            &MmioRegisterError::ProxyDeviceCreation(ref e) => write!(f, "failed to create proxy device: {}", e),
+            &MmioRegisterError::IrqsExhausted => write!(f, "no more IRQs are available"),
+            &MmioRegisterError::AddrsExhausted => write!(f, "no more addresses are available"),
+        }
+    }
+}
 
-    /// This creates a GuestMemory object for this VM
-    ///
-    /// * `mem_size` - Desired physical memory size in bytes for this VM
-    fn setup_memory(mem_size: u64) -> Result<GuestMemory>;
+/// Register a device to be used via MMIO transport.
+pub fn register_mmio(bus: &mut devices::Bus,
+                     vm: &mut Vm,
+                     device: Box<devices::virtio::VirtioDevice>,
+                     jail: Option<Minijail>,
+                     resources: &mut SystemAllocator,
+                     cmdline: &mut kernel_cmdline::Cmdline)
+                     -> std::result::Result<(), MmioRegisterError> {
+    let irq = match resources.allocate_irq() {
+        None => return Err(MmioRegisterError::IrqsExhausted),
+        Some(i) => i,
+    };
 
-    /// The creates the interrupt controller device and optionally returns the fd for it.
-    /// Some architectures may not have a separate descriptor for the interrupt
-    /// controller, so they would return None even on success.
-    ///
-    /// # Arguments
-    ///
-    /// * `vm` - the vm object
-    fn create_irq_chip(vm: &kvm::Vm) -> Result<Option<File>>;
+    // List of FDs to keep open in the child after it forks.
+    let mut keep_fds: Vec<RawFd> = device.keep_fds();
+    syslog::push_fds(&mut keep_fds);
 
-    /// This returns the first page frame number for use by the balloon driver.
-    ///
-    /// # Arguments
-    ///
-    /// * `mem_size` - the size in bytes of physical ram for the guest
-    fn get_base_dev_pfn(mem_size: u64) -> u64;
+    let mmio_device = devices::virtio::MmioDevice::new((*vm.get_memory()).clone(),
+    device)
+        .map_err(MmioRegisterError::CreateMmioDevice)?;
+    let mmio_len = 0x1000; // TODO(dgreid) - configurable per arch?
+    let mmio_base = resources.allocate_mmio_addresses(mmio_len)
+        .ok_or(MmioRegisterError::AddrsExhausted)?;
+    for (i, queue_evt) in mmio_device.queue_evts().iter().enumerate() {
+        let io_addr = IoeventAddress::Mmio(mmio_base +
+                                           devices::virtio::NOTIFY_REG_OFFSET as u64);
+        vm.register_ioevent(&queue_evt, io_addr, i as u32)
+          .map_err(MmioRegisterError::RegisterIoevent)?;
+        keep_fds.push(queue_evt.as_raw_fd());
+    }
 
-    /// This returns a minimal kernel command for this architecture.
-    fn get_base_linux_cmdline() -> kernel_cmdline::Cmdline;
+    if let Some(interrupt_evt) = mmio_device.interrupt_evt() {
+        vm.register_irqfd(&interrupt_evt, irq)
+          .map_err(MmioRegisterError::RegisterIrqfd)?;
+        keep_fds.push(interrupt_evt.as_raw_fd());
+    }
 
-    /// Returns a system resource allocator.
-    fn get_resource_allocator(mem_size: u64, gpu_allocation: bool) -> SystemAllocator;
+    if let Some(jail) = jail {
+        let proxy_dev = devices::ProxyDevice::new(mmio_device, &jail, keep_fds)
+            .map_err(MmioRegisterError::ProxyDeviceCreation)?;
 
-    /// Sets up the IO bus for this platform
-    ///
-    /// # Arguments
-    ///
-    /// * - `vm` the vm object
-    /// * - `exit_evt` - the event fd object which should receive exit events
-    fn setup_io_bus(vm: &mut Vm, exit_evt: EventFd)
-                    -> Result<(devices::Bus, Arc<Mutex<devices::Serial>>)>;
+        bus.insert(Arc::new(Mutex::new(proxy_dev)), mmio_base, mmio_len, false).unwrap();
+    } else {
+        bus.insert(Arc::new(Mutex::new(mmio_device)), mmio_base, mmio_len, false).unwrap();
+    }
 
-    /// Configures the vcpu and should be called once per vcpu from the vcpu's thread.
-    ///
-    /// # Arguments
-    ///
-    /// * `guest_mem` - The memory to be used by the guest.
-    /// * `kernel_load_offset` - Offset in bytes from `guest_mem` at which the
-    ///                          kernel starts.
-    /// * `kvm` - The /dev/kvm object that created vcpu.
-    /// * `vm` - The VM object associated with this VCPU.
-    /// * `vcpu` - The VCPU object to configure.
-    /// * `cpu_id` - The id of the given `vcpu`.
-    /// * `num_cpus` - Number of virtual CPUs the guest will have.
-    fn configure_vcpu(guest_mem: &GuestMemory,
-                      kvm: &Kvm,
-                      vm: &Vm,
-                      vcpu: &Vcpu,
-                      cpu_id: u64,
-                      num_cpus: u64)
-                      -> Result<()>;
+    cmdline
+        .insert("virtio_mmio.device",
+                &format!("4K@0x{:08x}:{}", mmio_base, irq))
+        .map_err(MmioRegisterError::Cmdline)?;
+
+    Ok(())
 }
diff --git a/src/linux.rs b/src/linux.rs
index 77eb2cb..a858129 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -4,18 +4,18 @@
 
 use std;
 use std::cmp::min;
-use std::ffi::{CString, CStr};
+use std::ffi::CStr;
 use std::fmt;
 use std::error;
-use std::fs::{File, OpenOptions, remove_file};
+use std::fs::{File, OpenOptions};
 use std::io::{self, Read, stdin};
 use std::mem;
-use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::os::unix::io::{FromRawFd, RawFd};
 use std::os::unix::net::UnixDatagram;
 use std::path::{Path, PathBuf};
 use std::str;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, Barrier};
+use std::sync::{Arc, Barrier};
 use std::thread;
 use std::time::Duration;
 use std::thread::JoinHandle;
@@ -27,43 +27,37 @@ use rand::distributions::{IndependentSample, Range};
 use byteorder::{ByteOrder, LittleEndian};
 use devices;
 use io_jail::{self, Minijail};
-use kernel_cmdline;
 use kvm::*;
 use net_util::Tap;
 use qcow::{self, QcowFile};
 use sys_util::*;
 use sys_util;
-use resources::SystemAllocator;
 use vhost;
 use vm_control::VmRequest;
 
 use Config;
 use DiskType;
+use VirtIoDeviceInfo;
 
-use arch::LinuxArch;
+use arch::{self, LinuxArch, RunnableLinuxVm, VirtioDeviceStub, VmComponents};
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use x86_64::X8664arch as Arch;
 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
 use aarch64::AArch64 as Arch;
 
+#[derive(Debug)]
 pub enum Error {
-    AddingArchDevs(Box<error::Error>),
     BalloonDeviceNew(devices::virtio::BalloonError),
     BlockDeviceNew(sys_util::Error),
     BlockSignal(sys_util::signal::Error),
+    BuildingVm(Box<error::Error>),
     CloneEventFd(sys_util::Error),
-    Cmdline(kernel_cmdline::Error),
     CreateEventFd(sys_util::Error),
-    CreateGuestMemory(Box<error::Error>),
-    CreateIrqChip(Box<error::Error>),
-    CreateKvm(sys_util::Error),
     CreatePollContext(sys_util::Error),
     CreateSignalFd(sys_util::SignalFdError),
     CreateSocket(io::Error),
     CreateTimerFd(sys_util::Error),
-    CreateVcpu(sys_util::Error),
-    CreateVm(Box<error::Error>),
     DeviceJail(io_jail::Error),
     DevicePivotRoot(io_jail::Error),
     Disk(io::Error),
@@ -80,15 +74,14 @@ pub enum Error {
     QcowDeviceCreate(qcow::Error),
     ReadLowmemAvailable(io::Error),
     ReadLowmemMargin(io::Error),
-    RegisterBalloon(MmioRegisterError),
-    RegisterBlock(MmioRegisterError),
-    RegisterGpu(MmioRegisterError),
-    RegisterNet(MmioRegisterError),
-    RegisterP9(MmioRegisterError),
-    RegisterRng(MmioRegisterError),
+    RegisterBalloon(arch::MmioRegisterError),
+    RegisterBlock(arch::MmioRegisterError),
+    RegisterGpu(arch::MmioRegisterError),
+    RegisterNet(arch::MmioRegisterError),
+    RegisterP9(arch::MmioRegisterError),
+    RegisterRng(arch::MmioRegisterError),
     RegisterSignalHandler(sys_util::Error),
-    RegisterVsock(MmioRegisterError),
-    RegisterWayland(MmioRegisterError),
+    RegisterWayland(arch::MmioRegisterError),
     ResetTimerFd(sys_util::Error),
     RngDeviceNew(devices::virtio::RngError),
     SettingGidMap(io_jail::Error),
@@ -99,34 +92,24 @@ pub enum Error {
     VhostNetDeviceNew(devices::virtio::vhost::Error),
     VhostVsockDeviceNew(devices::virtio::vhost::Error),
     WaylandDeviceNew(sys_util::Error),
-    SetupSystemMemory(Box<error::Error>),
-    ConfigureVcpu(Box<error::Error>),
     LoadKernel(Box<error::Error>),
-    SetupIoBus(Box<error::Error>),
-    SetupMMIOBus(Box<error::Error>),
 }
 
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            &Error::AddingArchDevs(ref e) => write!(f, "Failed to add arch devs {:?}", e),
             &Error::BalloonDeviceNew(ref e) => write!(f, "failed to create balloon: {:?}", e),
             &Error::BlockDeviceNew(ref e) => write!(f, "failed to create block device: {:?}", e),
             &Error::BlockSignal(ref e) => write!(f, "failed to block signal: {:?}", e),
+            &Error::BuildingVm(ref e) => {
+                write!(f, "The architecture failed to build the vm: {:?}", e)
+            }
             &Error::CloneEventFd(ref e) => write!(f, "failed to clone eventfd: {:?}", e),
-            &Error::Cmdline(ref e) => write!(f, "the given kernel command line was invalid: {}", e),
             &Error::CreateEventFd(ref e) => write!(f, "failed to create eventfd: {:?}", e),
-            &Error::CreateGuestMemory(ref e) => write!(f, "failed to create guest memory: {:?}", e),
-            &Error::CreateIrqChip(ref e) => {
-                write!(f, "failed to create in-kernel IRQ chip: {:?}", e)
-            }
-            &Error::CreateKvm(ref e) => write!(f, "failed to open /dev/kvm: {:?}", e),
             &Error::CreatePollContext(ref e) => write!(f, "failed to create poll context: {:?}", e),
             &Error::CreateSignalFd(ref e) => write!(f, "failed to create signalfd: {:?}", e),
             &Error::CreateSocket(ref e) => write!(f, "failed to create socket: {}", e),
             &Error::CreateTimerFd(ref e) => write!(f, "failed to create timerfd: {}", e),
-            &Error::CreateVcpu(ref e) => write!(f, "failed to create VCPU: {:?}", e),
-            &Error::CreateVm(ref e) => write!(f, "failed to create KVM VM object: {:?}", e),
             &Error::DeviceJail(ref e) => write!(f, "failed to jail device: {}", e),
             &Error::DevicePivotRoot(ref e) => write!(f, "failed to pivot root device: {}", e),
             &Error::Disk(ref e) => write!(f, "failed to load disk image: {}", e),
@@ -166,9 +149,6 @@ impl fmt::Display for Error {
             &Error::RegisterSignalHandler(ref e) => {
                 write!(f, "error registering signal handler: {:?}", e)
             }
-            &Error::RegisterVsock(ref e) => {
-                write!(f, "error registering virtual socket device: {:?}", e)
-            }
             &Error::RegisterWayland(ref e) => write!(f, "error registering wayland device: {}", e),
             &Error::ResetTimerFd(ref e) => write!(f, "failed to reset timerfd: {}", e),
             &Error::RngDeviceNew(ref e) => write!(f, "failed to set up rng: {:?}", e),
@@ -186,44 +166,28 @@ impl fmt::Display for Error {
             &Error::WaylandDeviceNew(ref e) => {
                 write!(f, "failed to create wayland device: {:?}", e)
             }
-            &Error::SetupSystemMemory(ref e) => write!(f, "error setting up system memory: {}", e),
-            &Error::ConfigureVcpu(ref e) => write!(f, "failed to configure vcpu: {}", e),
             &Error::LoadKernel(ref e) => write!(f, "failed to load kernel: {}", e),
-            &Error::SetupIoBus(ref e) => write!(f, "failed to setup iobus: {}", e),
-            &Error::SetupMMIOBus(ref e) => write!(f, "failed to setup mmio bus: {}", e),
         }
     }
 }
 
-type Result<T> = std::result::Result<T, Error>;
-
-struct UnlinkUnixDatagram(UnixDatagram);
-impl AsRef<UnixDatagram> for UnlinkUnixDatagram {
-    fn as_ref(&self) -> &UnixDatagram{
-        &self.0
-    }
-}
-impl Drop for UnlinkUnixDatagram {
-    fn drop(&mut self) {
-        if let Ok(addr) = self.0.local_addr() {
-            if let Some(path) = addr.as_pathname() {
-                if let Err(e) = remove_file(path) {
-                    warn!("failed to remove control socket file: {:?}", e);
-                }
-            }
-        }
+impl std::error::Error for Error {
+    fn description(&self) -> &str {
+        "Some device failure"
     }
 }
 
+type Result<T> = std::result::Result<T, Error>;
+
 // Verifies that |raw_fd| is actually owned by this process and duplicates it to ensure that
 // we have a unique handle to it.
-fn validate_raw_fd(raw_fd: RawFd) -> Result<RawFd> {
+fn validate_raw_fd(raw_fd: RawFd) -> std::result::Result<RawFd, Box<error::Error>> {
     // Checking that close-on-exec isn't set helps filter out FDs that were opened by
     // crosvm as all crosvm FDs are close on exec.
     // Safe because this doesn't modify any memory and we check the return value.
     let flags = unsafe { libc::fcntl(raw_fd, libc::F_GETFD) };
     if flags < 0 || (flags & libc::FD_CLOEXEC) != 0 {
-        return Err(Error::FailedCLOEXECCheck);
+        return Err(Box::new(Error::FailedCLOEXECCheck));
     }
 
     // Duplicate the fd to ensure that we don't accidentally close an fd previously
@@ -231,7 +195,7 @@ fn validate_raw_fd(raw_fd: RawFd) -> Result<RawFd> {
     // we check the return value.
     let dup_fd = unsafe { libc::fcntl(raw_fd, libc::F_DUPFD_CLOEXEC, 0) };
     if dup_fd < 0 {
-        return Err(Error::FailedToDupFd);
+        return Err(Box::new(Error::FailedToDupFd));
     }
     Ok(dup_fd as RawFd)
 }
@@ -266,135 +230,27 @@ fn create_base_minijail(root: &Path, seccomp_policy: &Path) -> Result<Minijail>
     Ok(j)
 }
 
-/// Errors for device manager.
-#[derive(Debug)]
-pub enum MmioRegisterError {
-    /// Could not create the mmio device to wrap a VirtioDevice.
-    CreateMmioDevice(sys_util::Error),
-    /// Failed to register ioevent with VM.
-    RegisterIoevent(sys_util::Error),
-    /// Failed to register irq eventfd with VM.
-    RegisterIrqfd(sys_util::Error),
-    /// Failed to initialize proxy device for jailed device.
-    ProxyDeviceCreation(devices::ProxyError),
-    /// Appending to kernel command line failed.
-    Cmdline(kernel_cmdline::Error),
-    /// No more IRQs are available.
-    IrqsExhausted,
-    /// No more MMIO space available.
-    AddrsExhausted,
-}
-
-impl fmt::Display for MmioRegisterError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            &MmioRegisterError::CreateMmioDevice(ref e) => write!(f, "failed to create mmio device: {:?}", e),
-            &MmioRegisterError::Cmdline(ref e) => {
-                write!(f, "unable to add device to kernel command line: {}", e)
-            }
-            &MmioRegisterError::RegisterIoevent(ref e) => {
-                write!(f, "failed to register ioevent to VM: {:?}", e)
-            }
-            &MmioRegisterError::RegisterIrqfd(ref e) => {
-                write!(f, "failed to register irq eventfd to VM: {:?}", e)
-            }
-            &MmioRegisterError::ProxyDeviceCreation(ref e) => write!(f, "failed to create proxy device: {}", e),
-            &MmioRegisterError::IrqsExhausted => write!(f, "no more IRQs are available"),
-            &MmioRegisterError::AddrsExhausted => write!(f, "no more addresses are available"),
-        }
-    }
-}
-
-/// Register a device to be used via MMIO transport.
-fn register_mmio(bus: &mut devices::Bus,
-                 vm: &mut Vm,
-                 device: Box<devices::virtio::VirtioDevice>,
-                 jail: Option<Minijail>,
-                 resources: &mut SystemAllocator,
-                 cmdline: &mut kernel_cmdline::Cmdline)
-                 -> std::result::Result<(), MmioRegisterError> {
-    let irq = match resources.allocate_irq() {
-        None => return Err(MmioRegisterError::IrqsExhausted),
-        Some(i) => i,
-    };
-
-    // List of FDs to keep open in the child after it forks.
-    let mut keep_fds: Vec<RawFd> = device.keep_fds();
-    syslog::push_fds(&mut keep_fds);
-
-    let mmio_device = devices::virtio::MmioDevice::new((*vm.get_memory()).clone(),
-    device)
-        .map_err(MmioRegisterError::CreateMmioDevice)?;
-    let mmio_len = 0x1000; // TODO(dgreid) - configurable per arch?
-    let mmio_base = resources.allocate_mmio_addresses(mmio_len)
-        .ok_or(MmioRegisterError::AddrsExhausted)?;
-    for (i, queue_evt) in mmio_device.queue_evts().iter().enumerate() {
-        let io_addr = IoeventAddress::Mmio(mmio_base +
-                                           devices::virtio::NOTIFY_REG_OFFSET as u64);
-        vm.register_ioevent(&queue_evt, io_addr, i as u32)
-          .map_err(MmioRegisterError::RegisterIoevent)?;
-        keep_fds.push(queue_evt.as_raw_fd());
-    }
-
-    if let Some(interrupt_evt) = mmio_device.interrupt_evt() {
-        vm
-            .register_irqfd(&interrupt_evt, irq)
-            .map_err(MmioRegisterError::RegisterIrqfd)?;
-        keep_fds.push(interrupt_evt.as_raw_fd());
-    }
-
-    if let Some(jail) = jail {
-        let proxy_dev = devices::ProxyDevice::new(mmio_device, &jail, keep_fds)
-            .map_err(MmioRegisterError::ProxyDeviceCreation)?;
-
-        bus
-            .insert(Arc::new(Mutex::new(proxy_dev)),
-            mmio_base,
-            mmio_len,
-            false)
-            .unwrap();
-    } else {
-        bus
-            .insert(Arc::new(Mutex::new(mmio_device)),
-            mmio_base,
-            mmio_len,
-            false)
-            .unwrap();
-    }
-
-    cmdline
-        .insert("virtio_mmio.device",
-                &format!("4K@0x{:08x}:{}", mmio_base, irq))
-        .map_err(MmioRegisterError::Cmdline)?;
-
-    Ok(())
-}
-
-fn setup_mmio_bus(cfg: &Config,
-                  _exit_evt: EventFd,
-                  vm: &mut Vm,
-                  mem: &GuestMemory,
-                  cmdline: &mut kernel_cmdline::Cmdline,
-                  control_sockets: &mut Vec<UnlinkUnixDatagram>,
-                  balloon_device_socket: UnixDatagram,
-                  resources: &mut SystemAllocator)
-                  -> Result<devices::Bus> {
+fn create_virtio_devs(cfg: VirtIoDeviceInfo,
+                      mem: &GuestMemory,
+                      _exit_evt: &EventFd,
+                      wayland_device_socket: UnixDatagram,
+                      balloon_device_socket: UnixDatagram)
+                      -> std::result::Result<Vec<VirtioDeviceStub>, Box<error::Error>> {
     static DEFAULT_PIVOT_ROOT: &'static str = "/var/empty";
-    let mut bus = devices::Bus::new();
 
-    Arch::add_arch_devs(vm, &mut bus).map_err(Error::AddingArchDevs)?;
+    let mut devs = Vec::new();
 
     // An empty directory for jailed device's pivot root.
     let empty_root_path = Path::new(DEFAULT_PIVOT_ROOT);
     if cfg.multiprocess && !empty_root_path.exists() {
-        return Err(Error::NoVarEmpty);
+        return Err(Box::new(Error::NoVarEmpty));
     }
 
     for disk in &cfg.disks {
         // Special case '/proc/self/fd/*' paths. The FD is already open, just use it.
         let mut raw_image: File = if disk.path.parent() == Some(Path::new("/proc/self/fd")) {
             if !disk.path.is_file() {
-                return Err(Error::InvalidFdPath);
+                return Err(Box::new(Error::InvalidFdPath));
             }
             let raw_fd = disk.path.file_name()
                 .and_then(|fd_osstr| fd_osstr.to_str())
@@ -437,8 +293,7 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        register_mmio(&mut bus, vm, block_box, jail, resources, cmdline)
-            .map_err(Error::RegisterBlock)?;
+        devs.push(VirtioDeviceStub {dev: block_box, jail});
     }
 
     let rng_box = Box::new(devices::virtio::Rng::new().map_err(Error::RngDeviceNew)?);
@@ -448,8 +303,7 @@ fn setup_mmio_bus(cfg: &Config,
     } else {
         None
     };
-    register_mmio(&mut bus, vm, rng_box, rng_jail, resources, cmdline)
-        .map_err(Error::RegisterRng)?;
+    devs.push(VirtioDeviceStub {dev: rng_box, jail: rng_jail});
 
     let balloon_box = Box::new(devices::virtio::Balloon::new(balloon_device_socket)
                                    .map_err(Error::BalloonDeviceNew)?);
@@ -459,8 +313,7 @@ fn setup_mmio_bus(cfg: &Config,
     } else {
         None
     };
-    register_mmio(&mut bus, vm, balloon_box, balloon_jail, resources, cmdline)
-        .map_err(Error::RegisterBalloon)?;
+    devs.push(VirtioDeviceStub {dev: balloon_box, jail: balloon_jail});
 
     // We checked above that if the IP is defined, then the netmask is, too.
     if let Some(tap_fd) = cfg.tap_fd {
@@ -477,8 +330,7 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        register_mmio(&mut bus, vm, net_box, jail, resources, cmdline)
-            .map_err(Error::RegisterNet)?;
+        devs.push(VirtioDeviceStub {dev: net_box, jail});
     } else if let Some(host_ip) = cfg.host_ip {
         if let Some(netmask) = cfg.netmask {
             if let Some(mac_address) = cfg.mac_address {
@@ -505,8 +357,7 @@ fn setup_mmio_bus(cfg: &Config,
                     None
                 };
 
-                register_mmio(&mut bus, vm, net_box, jail, resources, cmdline)
-                    .map_err(Error::RegisterNet)?;
+                devs.push(VirtioDeviceStub {dev: net_box, jail});
             }
         }
     }
@@ -514,14 +365,12 @@ fn setup_mmio_bus(cfg: &Config,
     if let Some(wayland_socket_path) = cfg.wayland_socket_path.as_ref() {
         let jailed_wayland_path = Path::new("/wayland-0");
 
-        let (host_socket, device_socket) = UnixDatagram::pair().map_err(Error::CreateSocket)?;
-        control_sockets.push(UnlinkUnixDatagram(host_socket));
         let wl_box = Box::new(devices::virtio::Wl::new(if cfg.multiprocess {
                                                            &jailed_wayland_path
                                                        } else {
                                                            wayland_socket_path.as_path()
                                                        },
-                                                       device_socket)
+                                                       wayland_device_socket)
                                       .map_err(Error::WaylandDeviceNew)?);
 
         let jail = if cfg.multiprocess {
@@ -568,8 +417,10 @@ fn setup_mmio_bus(cfg: &Config,
         } else {
             None
         };
-        register_mmio(&mut bus, vm, wl_box, jail, resources, cmdline)
-            .map_err(Error::RegisterWayland)?;
+        devs.push(VirtioDeviceStub {
+                dev: wl_box,
+                jail,
+        });
     }
 
     if let Some(cid) = cfg.cid {
@@ -584,8 +435,7 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        register_mmio(&mut bus, vm, vsock_box, jail, resources, cmdline)
-            .map_err(Error::RegisterVsock)?;
+        devs.push(VirtioDeviceStub {dev: vsock_box, jail});
     }
 
     #[cfg(feature = "gpu")]
@@ -595,14 +445,13 @@ fn setup_mmio_bus(cfg: &Config,
                 Box::new(devices::virtio::Gpu::new(_exit_evt
                                                        .try_clone()
                                                        .map_err(Error::CloneEventFd)?));
-            let gpu_jail = if cfg.multiprocess {
+            let jail = if cfg.multiprocess {
                 error!("jail for virtio-gpu is unimplemented");
                 unimplemented!();
             } else {
                 None
             };
-            register_mmio(&mut bus, vm, gpu_box, gpu_jail, resources, cmdline)
-                .map_err(Error::RegisterGpu)?;
+            devs.push(VirtioDeviceStub {dev: gpu_box, jail});
         }
     }
 
@@ -649,22 +498,10 @@ fn setup_mmio_bus(cfg: &Config,
 
         let p9_box = Box::new(devices::virtio::P9::new(root, tag).map_err(Error::P9DeviceNew)?);
 
-        register_mmio(&mut bus, vm, p9_box, jail, resources, cmdline).map_err(Error::RegisterP9)?;
+        devs.push(VirtioDeviceStub {dev: p9_box, jail});
     }
 
-    Ok(bus)
-}
-
-fn setup_vcpu(kvm: &Kvm,
-              vm: &Vm,
-              cpu_id: u32,
-              vcpu_count: u32)
-              -> Result<Vcpu> {
-    let vcpu = Vcpu::new(cpu_id as libc::c_ulong, &kvm, &vm)
-        .map_err(Error::CreateVcpu)?;
-    Arch::configure_vcpu(vm.get_memory(), &kvm, &vm, &vcpu, cpu_id as u64, vcpu_count as u64).
-        map_err(Error::ConfigureVcpu)?;
-    Ok(vcpu)
+    Ok(devs)
 }
 
 fn setup_vcpu_signal_handler() -> Result<()> {
@@ -773,20 +610,57 @@ fn file_to_u64<P: AsRef<Path>>(path: P) -> io::Result<u64> {
     content.trim().parse().map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
 }
 
-struct RunnableLinuxVm {
-    pub vm: Vm,
-    pub control_sockets: Vec<UnlinkUnixDatagram>,
-    pub resources: SystemAllocator,
-    pub stdio_serial: Arc<Mutex<devices::Serial>>,
-    pub exit_evt: EventFd,
-    pub sigchld_fd: SignalFd,
-    pub kill_signaled: Arc<AtomicBool>,
-    pub vcpu_handles: Vec<JoinHandle<()>>,
-    pub balloon_host_socket: UnixDatagram,
-    pub irq_chip: Option<File>,
+pub fn run_config(cfg: Config) -> Result<()> {
+    if cfg.virtio_dev_info.multiprocess {
+        // Printing something to the syslog before entering minijail so that libc's syslogger has a
+        // chance to open files necessary for its operation, like `/etc/localtime`. After jailing,
+        // access to those files will not be possible.
+        info!("crosvm entering multiprocess mode");
+    }
+
+    let pci_devices = devices::PciDeviceList::new();
+
+    // Masking signals is inherently dangerous, since this can persist across clones/execs. Do this
+    // before any jailed devices have been spawned, so that we can catch any of them that fail very
+    // quickly.
+    let sigchld_fd = SignalFd::new(libc::SIGCHLD).map_err(Error::CreateSignalFd)?;
+
+    let components = VmComponents {
+        pci_devices,
+        memory_mb: (cfg.memory.unwrap_or(256) << 20) as u64,
+        vcpu_count: cfg.vcpu_count.unwrap_or(1),
+        kernel_image: File::open(cfg.kernel_path.as_path())
+            .map_err(|e| Error::OpenKernel(cfg.kernel_path.clone(), e))?,
+        extra_kernel_params: cfg.params,
+        wayland_dmabuf: cfg.virtio_dev_info.wayland_dmabuf,
+    };
+
+    let mut control_sockets = Vec::new();
+    if let Some(ref path_string) = cfg.socket_path {
+        let path = Path::new(path_string);
+        let dgram = UnixDatagram::bind(path).map_err(Error::CreateSocket)?;
+        control_sockets.push(UnlinkUnixDatagram(dgram));
+    };
+    let (wayland_host_socket, wayland_device_socket) = UnixDatagram::pair()
+        .map_err(Error::CreateSocket)?;
+    control_sockets.push(UnlinkUnixDatagram(wayland_host_socket));
+    // Balloon gets a special socket so balloon requests can be forwarded from the main process.
+    let (balloon_host_socket, balloon_device_socket) = UnixDatagram::pair()
+        .map_err(Error::CreateSocket)?;
+
+    let virtio_dev_info = cfg.virtio_dev_info;
+    let linux = Arch::build_vm(components,
+                               |m, e| create_virtio_devs(virtio_dev_info, m, e,
+                                                         wayland_device_socket,
+                                                         balloon_device_socket))
+        .map_err(Error::BuildingVm)?;
+    run_control(linux, control_sockets, balloon_host_socket, sigchld_fd)
 }
 
-fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
+fn run_control(mut linux: RunnableLinuxVm,
+               control_sockets: Vec<UnlinkUnixDatagram>,
+               balloon_host_socket: UnixDatagram,
+               sigchld_fd: SignalFd) -> Result<()> {
     const MAX_VM_FD_RECV: usize = 1;
 
     // Paths to get the currently available memory and the low memory threshold.
@@ -831,8 +705,8 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
     if let Err(e) = poll_ctx.add(&stdin_handle, Token::Stdin) {
         warn!("failed to add stdin to poll context: {:?}", e);
     }
-    poll_ctx.add(&linux.sigchld_fd, Token::ChildSignal).map_err(Error::PollContextAdd)?;
-    for (index, socket) in linux.control_sockets.iter().enumerate() {
+    poll_ctx.add(&sigchld_fd, Token::ChildSignal).map_err(Error::PollContextAdd)?;
+    for (index, socket) in control_sockets.iter().enumerate() {
         poll_ctx.add(socket.as_ref(), Token::VmControl{ index }).map_err(Error::PollContextAdd)?;
     }
 
@@ -859,6 +733,22 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
     let freemem_jitter_secs = Range::new(0, 12);
     let interval_jitter_secs = Range::new(0, 6);
 
+    let mut vcpu_handles = Vec::with_capacity(linux.vcpus.len() as usize);
+    let vcpu_thread_barrier = Arc::new(Barrier::new((linux.vcpus.len() + 1) as usize));
+    let kill_signaled = Arc::new(AtomicBool::new(false));
+    setup_vcpu_signal_handler()?;
+    for (cpu_id, vcpu) in linux.vcpus.into_iter().enumerate() {
+        let handle = run_vcpu(vcpu,
+                              cpu_id as u32,
+                              vcpu_thread_barrier.clone(),
+                              linux.io_bus.clone(),
+                              linux.mmio_bus.clone(),
+                              linux.exit_evt.try_clone().map_err(Error::CloneEventFd)?,
+                              kill_signaled.clone())?;
+        vcpu_handles.push(handle);
+    }
+    vcpu_thread_barrier.wait();
+
     let mut scm = Scm::new(MAX_VM_FD_RECV);
 
     'poll: loop {
@@ -900,7 +790,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                 Token::ChildSignal => {
                     // Print all available siginfo structs, then exit the loop.
                     loop {
-                        let result = linux.sigchld_fd.read().map_err(Error::SignalFd)?;
+                        let result = sigchld_fd.read().map_err(Error::SignalFd)?;
                         if let Some(siginfo) = result {
                             error!("child {} died: signo {}, status {}, code {}",
                                    siginfo.ssi_pid,
@@ -936,7 +826,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                         };
                         let mut buf = [0u8; mem::size_of::<u64>()];
                         LittleEndian::write_u64(&mut buf, current_balloon_memory);
-                        if let Err(e) = linux.balloon_host_socket.send(&buf) {
+                        if let Err(e) = balloon_host_socket.send(&buf) {
                             warn!("failed to send memory value to balloon device: {}", e);
                         }
                     }
@@ -950,7 +840,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                         if current_balloon_memory != old_balloon_memory {
                             let mut buf = [0u8; mem::size_of::<u64>()];
                             LittleEndian::write_u64(&mut buf, current_balloon_memory);
-                            if let Err(e) = linux.balloon_host_socket.send(&buf) {
+                            if let Err(e) = balloon_host_socket.send(&buf) {
                                 warn!("failed to send memory value to balloon device: {}", e);
                             }
                         }
@@ -986,7 +876,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                     }
                 }
                 Token::VmControl { index } => {
-                    if let Some(socket) = linux.control_sockets.get(index as usize) {
+                    if let Some(socket) = control_sockets.get(index as usize) {
                         match VmRequest::recv(&mut scm, socket.as_ref()) {
                             Ok(request) => {
                                 let mut running = true;
@@ -994,7 +884,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                                     request.execute(&mut linux.vm,
                                                     &mut linux.resources,
                                                     &mut running,
-                                                    &linux.balloon_host_socket);
+                                                    &balloon_host_socket);
                                 if let Err(e) = response.send(&mut scm, socket.as_ref()) {
                                     error!("failed to send VmResponse: {:?}", e);
                                 }
@@ -1024,7 +914,7 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
                     Token::LowMemory => {},
                     Token::LowmemTimer => {},
                     Token::VmControl { index } => {
-                        if let Some(socket) = linux.control_sockets.get(index as usize) {
+                        if let Some(socket) = control_sockets.get(index as usize) {
                             let _ = poll_ctx.delete(socket.as_ref());
                         }
                     },
@@ -1035,8 +925,8 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
 
     // vcpu threads MUST see the kill signaled flag, otherwise they may
     // re-enter the VM.
-    linux.kill_signaled.store(true, Ordering::SeqCst);
-    for handle in linux.vcpu_handles {
+    kill_signaled.store(true, Ordering::SeqCst);
+    for handle in vcpu_handles {
         match handle.kill(SIGRTMIN() + 0) {
             Ok(_) => {
                 if let Err(e) = handle.join() {
@@ -1053,102 +943,3 @@ fn run_control(mut linux: RunnableLinuxVm) -> Result<()> {
 
     Ok(())
 }
-
-pub fn run_config(cfg: Config) -> Result<()> {
-    if cfg.multiprocess {
-        // Printing something to the syslog before entering minijail so that libc's syslogger has a
-        // chance to open files necessary for its operation, like `/etc/localtime`. After jailing,
-        // access to those files will not be possible.
-        info!("crosvm entering multiprocess mode");
-    }
-
-
-    // Masking signals is inherently dangerous, since this can persist across clones/execs. Do this
-    // before any jailed devices have been spawned, so that we can catch any of them that fail very
-    // quickly.
-    let sigchld_fd = SignalFd::new(libc::SIGCHLD).map_err(Error::CreateSignalFd)?;
-
-    let mut control_sockets = Vec::new();
-    if let Some(ref path) = cfg.socket_path {
-        let path = Path::new(path);
-        let control_socket = UnixDatagram::bind(path).map_err(Error::CreateSocket)?;
-        control_sockets.push(UnlinkUnixDatagram(control_socket));
-    }
-
-    let kill_signaled = Arc::new(AtomicBool::new(false));
-    let exit_evt = EventFd::new().map_err(Error::CreateEventFd)?;
-
-    let mem_size = cfg.memory.unwrap_or(256) << 20;
-    let mut resources = Arch::get_resource_allocator(mem_size as u64, cfg.wayland_dmabuf);
-    let mem = Arch::setup_memory(mem_size as u64).map_err(|e| Error::CreateGuestMemory(e))?;
-    let kvm = Kvm::new().map_err(Error::CreateKvm)?;
-    let mut vm = Arch::create_vm(&kvm, mem.clone()).map_err(|e| Error::CreateVm(e))?;
-
-    let vcpu_count = cfg.vcpu_count.unwrap_or(1);
-    let mut vcpu_handles = Vec::with_capacity(vcpu_count as usize);
-    let vcpu_thread_barrier = Arc::new(Barrier::new((vcpu_count + 1) as usize));
-    let mut vcpus = Vec::with_capacity(vcpu_count as usize);
-    for cpu_id in 0..vcpu_count {
-        let vcpu = setup_vcpu(&kvm, &vm, cpu_id, vcpu_count)?;
-        vcpus.push(vcpu);
-    }
-
-    let irq_chip = Arch::create_irq_chip(&vm).map_err(|e| Error::CreateIrqChip(e))?;
-    let mut cmdline = Arch::get_base_linux_cmdline();
-    let (io_bus, stdio_serial) = Arch::setup_io_bus(&mut vm,
-                                                    exit_evt.try_clone().
-                                                    map_err(Error::CloneEventFd)?).
-        map_err(|e| Error::SetupIoBus(e))?;
-
-    let (balloon_host_socket, balloon_device_socket) = UnixDatagram::pair()
-        .map_err(Error::CreateSocket)?;
-    let mmio_bus = setup_mmio_bus(&cfg,
-                                  exit_evt.try_clone().map_err(Error::CloneEventFd)?,
-                                  &mut vm,
-                                  &mem,
-                                  &mut cmdline,
-                                  &mut control_sockets,
-                                  balloon_device_socket,
-                                  &mut resources)?;
-
-    for param in &cfg.params {
-        cmdline.insert_str(&param).map_err(Error::Cmdline)?;
-    }
-
-    let mut kernel_image = File::open(cfg.kernel_path.as_path())
-        .map_err(|e| Error::OpenKernel(cfg.kernel_path.clone(), e))?;
-
-    // separate out load_kernel from other setup to get a specific error for
-    // kernel loading
-    Arch::load_kernel(&mem, &mut kernel_image).map_err(|e| Error::LoadKernel(e))?;
-    Arch::setup_system_memory(&mem, mem_size as u64, vcpu_count,
-                              &CString::new(cmdline).unwrap(), Vec::new())
-        .map_err(|e| Error::SetupSystemMemory(e))?;
-
-    setup_vcpu_signal_handler()?;
-    for (cpu_id, vcpu) in vcpus.into_iter().enumerate() {
-        let handle = run_vcpu(vcpu,
-                              cpu_id as u32,
-                              vcpu_thread_barrier.clone(),
-                              io_bus.clone(),
-                              mmio_bus.clone(),
-                              exit_evt.try_clone().map_err(Error::CloneEventFd)?,
-                              kill_signaled.clone())?;
-        vcpu_handles.push(handle);
-    }
-    vcpu_thread_barrier.wait();
-
-    let linux = RunnableLinuxVm {
-        vm,
-        control_sockets,
-        resources,
-        stdio_serial,
-        exit_evt,
-        sigchld_fd,
-        kill_signaled,
-        vcpu_handles,
-        balloon_host_socket,
-        irq_chip,
-    };
-    run_control(linux)
-}
diff --git a/src/main.rs b/src/main.rs
index 576ae20..bd2b632 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -66,52 +66,60 @@ struct DiskOption {
     disk_type: DiskType,
 }
 
-pub struct Config {
+/// Contains all the info needed to create the system's virtio devices.
+/// TODO(dgreid) - remove this once all devices are PCI based instead of MMIO.
+pub struct VirtIoDeviceInfo {
     disks: Vec<DiskOption>,
-    vcpu_count: Option<u32>,
-    memory: Option<usize>,
-    kernel_path: PathBuf,
-    params: Vec<String>,
     host_ip: Option<net::Ipv4Addr>,
     netmask: Option<net::Ipv4Addr>,
     mac_address: Option<net_util::MacAddress>,
     vhost_net: bool,
     tap_fd: Option<RawFd>,
+    cid: Option<u64>,
     wayland_socket_path: Option<PathBuf>,
     wayland_dmabuf: bool,
-    socket_path: Option<PathBuf>,
     shared_dirs: Vec<(PathBuf, String)>,
     multiprocess: bool,
     seccomp_policy_dir: PathBuf,
-    cid: Option<u64>,
     gpu: bool,
+}
+
+pub struct Config {
+    vcpu_count: Option<u32>,
+    memory: Option<usize>,
+    kernel_path: PathBuf,
+    params: Vec<String>,
+    socket_path: Option<PathBuf>,
     plugin: Option<PathBuf>,
     plugin_root: Option<PathBuf>,
+    virtio_dev_info: VirtIoDeviceInfo,
 }
 
 impl Default for Config {
     fn default() -> Config {
         Config {
-            disks: Vec::new(),
             vcpu_count: None,
             memory: None,
             kernel_path: PathBuf::default(),
             params: Vec::new(),
-            host_ip: None,
-            netmask: None,
-            mac_address: None,
-            vhost_net: false,
-            tap_fd: None,
-            wayland_socket_path: None,
-            wayland_dmabuf: false,
             socket_path: None,
-            multiprocess: !cfg!(feature = "default-no-sandbox"),
-            shared_dirs: Vec::new(),
-            seccomp_policy_dir: PathBuf::from(SECCOMP_POLICY_DIR),
-            cid: None,
-            gpu: false,
             plugin: None,
             plugin_root: None,
+            virtio_dev_info: VirtIoDeviceInfo {
+                disks: Vec::new(),
+                host_ip: None,
+                netmask: None,
+                mac_address: None,
+                vhost_net: false,
+                tap_fd: None,
+                cid: None,
+                gpu: false,
+                wayland_socket_path: None,
+                wayland_dmabuf: false,
+                shared_dirs: Vec::new(),
+                multiprocess: !cfg!(feature = "default-no-sandbox"),
+                seccomp_policy_dir: PathBuf::from(SECCOMP_POLICY_DIR),
+            },
         }
     }
 }
@@ -206,14 +214,14 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                            });
             }
             if name == "root" {
-                if cfg.disks.len() >= 26 {
+                if cfg.virtio_dev_info.disks.len() >= 26 {
                     return Err(argument::Error::TooManyArguments("ran out of letters for to assign to root disk".to_owned()));
                 }
                 cfg.params
                     .push(format!("root=/dev/vd{} ro",
-                                  char::from('a' as u8 + cfg.disks.len() as u8)));
+                                  char::from('a' as u8 + cfg.virtio_dev_info.disks.len() as u8)));
             }
-            cfg.disks
+            cfg.virtio_dev_info.disks
                 .push(DiskOption {
                           path: disk_path,
                           read_only: !name.starts_with("rw"),
@@ -225,10 +233,10 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                       });
         }
         "host_ip" => {
-            if cfg.host_ip.is_some() {
+            if cfg.virtio_dev_info.host_ip.is_some() {
                 return Err(argument::Error::TooManyArguments("`host_ip` already given".to_owned()));
             }
-            cfg.host_ip =
+            cfg.virtio_dev_info.host_ip =
                 Some(value
                          .unwrap()
                          .parse()
@@ -240,10 +248,10 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                                   })?)
         }
         "netmask" => {
-            if cfg.netmask.is_some() {
+            if cfg.virtio_dev_info.netmask.is_some() {
                 return Err(argument::Error::TooManyArguments("`netmask` already given".to_owned()));
             }
-            cfg.netmask =
+            cfg.virtio_dev_info.netmask =
                 Some(value
                          .unwrap()
                          .parse()
@@ -255,10 +263,10 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                                   })?)
         }
         "mac" => {
-            if cfg.mac_address.is_some() {
+            if cfg.virtio_dev_info.mac_address.is_some() {
                 return Err(argument::Error::TooManyArguments("`mac` already given".to_owned()));
             }
-            cfg.mac_address =
+            cfg.virtio_dev_info.mac_address =
                 Some(value
                          .unwrap()
                          .parse()
@@ -270,7 +278,7 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                                   })?)
         }
         "wayland-sock" => {
-            if cfg.wayland_socket_path.is_some() {
+            if cfg.virtio_dev_info.wayland_socket_path.is_some() {
                 return Err(argument::Error::TooManyArguments("`wayland-sock` already given"
                                                                  .to_owned()));
             }
@@ -281,11 +289,11 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                                expected: "Wayland socket does not exist",
                            });
             }
-            cfg.wayland_socket_path = Some(wayland_socket_path);
+            cfg.virtio_dev_info.wayland_socket_path = Some(wayland_socket_path);
         }
         #[cfg(feature = "wl-dmabuf")]
         "wayland-dmabuf" => {
-            cfg.wayland_dmabuf = true
+            cfg.virtio_dev_info.wayland_dmabuf = true
         }
         "socket" => {
             if cfg.socket_path.is_some() {
@@ -304,16 +312,16 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
             cfg.socket_path = Some(socket_path);
         }
         "multiprocess" => {
-            cfg.multiprocess = true;
+            cfg.virtio_dev_info.multiprocess = true;
         }
         "disable-sandbox" => {
-            cfg.multiprocess = false;
+            cfg.virtio_dev_info.multiprocess = false;
         }
         "cid" => {
-            if cfg.cid.is_some() {
+            if cfg.virtio_dev_info.cid.is_some() {
                 return Err(argument::Error::TooManyArguments("`cid` alread given".to_owned()));
             }
-            cfg.cid = Some(value.unwrap().parse().map_err(|_| {
+            cfg.virtio_dev_info.cid = Some(value.unwrap().parse().map_err(|_| {
                 argument::Error::InvalidValue {
                     value: value.unwrap().to_owned(),
                     expected: "this value for `cid` must be an unsigned integer",
@@ -344,11 +352,11 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                 });
             }
 
-            cfg.shared_dirs.push((src, tag));
+            cfg.virtio_dev_info.shared_dirs.push((src, tag));
         }
         "seccomp-policy-dir" => {
             // `value` is Some because we are in this match so it's safe to unwrap.
-            cfg.seccomp_policy_dir = PathBuf::from(value.unwrap());
+            cfg.virtio_dev_info.seccomp_policy_dir = PathBuf::from(value.unwrap());
         },
         "plugin" => {
             if !cfg.kernel_path.as_os_str().is_empty() {
@@ -369,13 +377,13 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
             cfg.plugin_root = Some(PathBuf::from(value.unwrap().to_owned()));
         },
         "vhost-net" => {
-            cfg.vhost_net = true
+            cfg.virtio_dev_info.vhost_net = true
         },
         "tap-fd" => {
-            if cfg.tap_fd.is_some() {
+            if cfg.virtio_dev_info.tap_fd.is_some() {
                 return Err(argument::Error::TooManyArguments("`tap-fd` alread given".to_owned()));
             }
-            cfg.tap_fd = Some(value.unwrap().parse().map_err(|_| {
+            cfg.virtio_dev_info.tap_fd = Some(value.unwrap().parse().map_err(|_| {
                 argument::Error::InvalidValue {
                     value: value.unwrap().to_owned(),
                     expected: "this value for `tap-fd` must be an unsigned integer",
@@ -383,7 +391,7 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
             })?);
         }
         "gpu" => {
-            cfg.gpu = true;
+            cfg.virtio_dev_info.gpu = true;
         }
         "help" => return Err(argument::Error::PrintHelp),
         _ => unreachable!(),
@@ -449,21 +457,24 @@ fn run_vm(args: std::env::Args) -> std::result::Result<(), ()> {
         if cfg.kernel_path.as_os_str().is_empty() && cfg.plugin.is_none() {
             return Err(argument::Error::ExpectedArgument("`KERNEL`".to_owned()));
         }
-        if cfg.host_ip.is_some() || cfg.netmask.is_some() || cfg.mac_address.is_some() {
-            if cfg.host_ip.is_none() {
+        if cfg.virtio_dev_info.host_ip.is_some() || cfg.virtio_dev_info.netmask.is_some()
+                || cfg.virtio_dev_info.mac_address.is_some() {
+            if cfg.virtio_dev_info.host_ip.is_none() {
                 return Err(argument::Error::ExpectedArgument("`host_ip` missing from network config".to_owned()));
             }
-            if cfg.netmask.is_none() {
+            if cfg.virtio_dev_info.netmask.is_none() {
                 return Err(argument::Error::ExpectedArgument("`netmask` missing from network config".to_owned()));
             }
-            if cfg.mac_address.is_none() {
+            if cfg.virtio_dev_info.mac_address.is_none() {
                 return Err(argument::Error::ExpectedArgument("`mac` missing from network config".to_owned()));
             }
         }
         if cfg.plugin_root.is_some() && cfg.plugin.is_none() {
             return Err(argument::Error::ExpectedArgument("`plugin-root` requires `plugin`".to_owned()));
         }
-        if cfg.tap_fd.is_some() && (cfg.host_ip.is_some() || cfg.netmask.is_some() || cfg.mac_address.is_some()) {
+        if cfg.virtio_dev_info.tap_fd.is_some() && (cfg.virtio_dev_info.host_ip.is_some() ||
+                                                    cfg.virtio_dev_info.netmask.is_some() ||
+                                                    cfg.virtio_dev_info.mac_address.is_some()) {
             return Err(argument::Error::TooManyArguments(
                 "`tap_fd` and any of `host_ip`, `netmask`, or `mac` are mutually exclusive".to_owned()));
         }
diff --git a/src/plugin/mod.rs b/src/plugin/mod.rs
index 750f22b..ef55afe 100644
--- a/src/plugin/mod.rs
+++ b/src/plugin/mod.rs
@@ -425,7 +425,7 @@ pub fn run_config(cfg: Config) -> Result<()> {
     // quickly.
     let sigchld_fd = SignalFd::new(SIGCHLD).map_err(Error::CreateSignalFd)?;
 
-    let jail = if cfg.multiprocess {
+    let jail = if cfg.virtio_dev_info.multiprocess {
         // An empty directory for jailed plugin pivot root.
         let root_path = match cfg.plugin_root {
             Some(ref dir) => Path::new(dir),
@@ -444,7 +444,7 @@ pub fn run_config(cfg: Config) -> Result<()> {
             return Err(Error::RootNotDir);
         }
 
-        let policy_path = cfg.seccomp_policy_dir.join("plugin.policy");
+        let policy_path = cfg.virtio_dev_info.seccomp_policy_dir.join("plugin.policy");
         let jail = create_plugin_jail(root_path, &policy_path)?;
         Some(jail)
     } else {
@@ -452,9 +452,9 @@ pub fn run_config(cfg: Config) -> Result<()> {
     };
 
     let mut tap_opt: Option<Tap> = None;
-    if let Some(host_ip) = cfg.host_ip {
-        if let Some(netmask) = cfg.netmask {
-            if let Some(mac_address) = cfg.mac_address {
+    if let Some(host_ip) = cfg.virtio_dev_info.host_ip {
+        if let Some(netmask) = cfg.virtio_dev_info.netmask {
+            if let Some(mac_address) = cfg.virtio_dev_info.mac_address {
                 let tap = Tap::new(false).map_err(Error::TapOpen)?;
                 tap.set_ip_addr(host_ip).map_err(Error::TapSetIp)?;
                 tap.set_netmask(netmask).map_err(Error::TapSetNetmask)?;
diff --git a/sys_util/src/lib.rs b/sys_util/src/lib.rs
index c8f38de..55861af 100644
--- a/sys_util/src/lib.rs
+++ b/sys_util/src/lib.rs
@@ -63,8 +63,9 @@ pub use signalfd::Error as SignalFdError;
 pub use write_zeroes::WriteZeroes as WriteZeroes;
 
 use std::ffi::CStr;
-use std::fs::File;
+use std::fs::{File, remove_file};
 use std::os::unix::io::{AsRawFd, FromRawFd};
+use std::os::unix::net::UnixDatagram;
 use std::ptr;
 
 use libc::{kill, syscall, sysconf, waitpid, pipe2, c_long, pid_t, uid_t, gid_t, _SC_PAGESIZE,
@@ -271,3 +272,23 @@ pub fn pipe(close_on_exec: bool) -> Result<(File, File)> {
         })
     }
 }
+
+/// Used to attempt to clean up a named pipe after it is no longer used.
+pub struct UnlinkUnixDatagram(pub UnixDatagram);
+impl AsRef<UnixDatagram> for UnlinkUnixDatagram {
+    fn as_ref(&self) -> &UnixDatagram {
+        &self.0
+    }
+}
+impl Drop for UnlinkUnixDatagram {
+    fn drop(&mut self) {
+        if let Ok(addr) = self.0.local_addr() {
+            if let Some(path) = addr.as_pathname() {
+                if let Err(e) = remove_file(path) {
+                    warn!("failed to remove control socket file: {:?}", e);
+                }
+            }
+        }
+    }
+}
+
diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs
index 32f9a0c..fe7de3b 100644
--- a/x86_64/src/lib.rs
+++ b/x86_64/src/lib.rs
@@ -63,10 +63,11 @@ use std::result;
 use std::error::{self, Error as X86Error};
 use std::fmt::{self, Display};
 use std::fs::File;
-use std::ffi::CStr;
+use std::ffi::{CStr, CString};
+use std::io::{self, stdout};
 use std::sync::{Arc, Mutex};
-use std::io::stdout;
 
+use arch::{RunnableLinuxVm, VirtioDeviceStub, VmComponents};
 use bootparam::boot_params;
 use bootparam::E820_RAM;
 use devices::PciInterruptPin;
@@ -76,16 +77,30 @@ use kvm::*;
 
 #[derive(Debug)]
 pub enum Error {
+    /// Error Adding a PCI device.
+    AddPciDev(devices::PciRootError),
     /// Error configuring the system
     ConfigureSystem,
     /// Unable to clone an EventFd
     CloneEventFd(sys_util::Error),
+    /// Error creating kernel command line.
+    Cmdline(kernel_cmdline::Error),
     /// Unable to make an EventFd
     CreateEventFd(sys_util::Error),
+    /// Unable to create Kvm.
+    CreateKvm(sys_util::Error),
+    /// Unable to create a PciRoot hub.
+    CreatePciRoot(devices::PciRootError),
+    /// Unable to create socket.
+    CreateSocket(io::Error),
+    /// Unable to create Vcpu.
+    CreateVcpu(sys_util::Error),
     /// The kernel extends past the end of RAM
     KernelOffsetPastEnd,
     /// Error registering an IrqFd
     RegisterIrqfd(sys_util::Error),
+    /// Couldn't register virtio socket.
+    RegisterVsock(arch::MmioRegisterError),
     LoadCmdline(kernel_loader::Error),
     LoadKernel(kernel_loader::Error),
     /// Error writing the zero page of guest memory.
@@ -99,12 +114,19 @@ pub enum Error {
 impl error::Error for Error {
     fn description(&self) -> &str {
         match self {
+            &Error::AddPciDev(_) => "Failed to add device to PCI",
             &Error::ConfigureSystem => "Error configuring the system",
             &Error::CloneEventFd(_) => "Unable to clone an EventFd",
+            &Error::Cmdline(_) => "the given kernel command line was invalid",
             &Error::CreateEventFd(_) => "Unable to make an EventFd",
+            &Error::CreateKvm(_) => "failed to open /dev/kvm",
+            &Error::CreatePciRoot(_) => "failed to create a PCI root hub",
+            &Error::CreateSocket(_) => "failed to create socket",
+            &Error::CreateVcpu(_) => "failed to create VCPU",
             &Error::KernelOffsetPastEnd =>
                 "The kernel extends past the end of RAM",
             &Error::RegisterIrqfd(_) => "Error registering an IrqFd",
+            &Error::RegisterVsock(_) => "error registering virtual socket device",
             &Error::LoadCmdline(_) => "Error Loading command line",
             &Error::LoadKernel(_) => "Error Loading Kernel",
             &Error::ZeroPageSetup =>
@@ -231,6 +253,76 @@ fn arch_memory_regions(size: u64) -> Vec<(GuestAddress, u64)> {
 }
 
 impl arch::LinuxArch for X8664arch {
+    fn build_vm<F>(mut components: VmComponents, virtio_devs: F) -> Result<RunnableLinuxVm>
+        where
+            F: FnOnce(&GuestMemory, &EventFd) -> Result<Vec<VirtioDeviceStub>>
+    {
+        let mut resources = Self::get_resource_allocator(components.memory_mb,
+                                                         components.wayland_dmabuf);
+        let mem = Self::setup_memory(components.memory_mb)?;
+        let kvm = Kvm::new().map_err(Error::CreateKvm)?;
+        let mut vm = Self::create_vm(&kvm, mem.clone())?;
+
+        let vcpu_count = components.vcpu_count;
+        let mut vcpus = Vec::with_capacity(vcpu_count as usize);
+        for cpu_id in 0..vcpu_count {
+            let vcpu = Vcpu::new(cpu_id as libc::c_ulong, &kvm, &vm)
+                .map_err(Error::CreateVcpu)?;
+            Self::configure_vcpu(vm.get_memory(), &kvm, &vm, &vcpu,
+                                 cpu_id as u64, vcpu_count as u64)?;
+            vcpus.push(vcpu);
+        }
+
+        let irq_chip = Self::create_irq_chip(&vm)?;
+        let mut cmdline = Self::get_base_linux_cmdline();
+
+        let mut mmio_bus = devices::Bus::new();
+
+        let (pci, pci_irqs) = components.pci_devices.generate_root(&mut mmio_bus, &mut resources)
+            .map_err(Error::CreatePciRoot)?;
+        let pci_bus = Arc::new(Mutex::new(pci));
+
+        let exit_evt = EventFd::new().map_err(Error::CreateEventFd)?;
+        let (io_bus, stdio_serial) = Self::setup_io_bus(
+            &mut vm,
+            exit_evt.try_clone().map_err(Error::CloneEventFd)?,
+            Some(pci_bus.clone()))?;
+
+
+        // Create a list of mmio devices to be added.
+        let mmio_devs = virtio_devs(&mem, &exit_evt)?;
+
+        for stub in mmio_devs {
+            arch::register_mmio(&mut mmio_bus, &mut vm, stub.dev, stub.jail,
+                                &mut resources, &mut cmdline)
+                .map_err(Error::RegisterVsock)?;
+        }
+
+        for param in components.extra_kernel_params {
+            cmdline.insert_str(&param).map_err(Error::Cmdline)?;
+        }
+
+        // separate out load_kernel from other setup to get a specific error for
+        // kernel loading
+        Self::load_kernel(&mem, &mut components.kernel_image)?;
+        Self::setup_system_memory(&mem, components.memory_mb, vcpu_count,
+                                  &CString::new(cmdline).unwrap(), pci_irqs)?;
+
+        Ok(RunnableLinuxVm {
+            vm,
+            kvm,
+            resources,
+            stdio_serial,
+            exit_evt,
+            vcpus,
+            irq_chip,
+            io_bus,
+            mmio_bus,
+        })
+    }
+}
+
+impl X8664arch {
     /// Loads the kernel from an open file.
     ///
     /// # Arguments
@@ -337,7 +429,7 @@ impl arch::LinuxArch for X8664arch {
     ///
     /// * - `vm` the vm object
     /// * - `exit_evt` - the event fd object which should receive exit events
-    fn setup_io_bus(vm: &mut Vm, exit_evt: EventFd)
+    fn setup_io_bus(vm: &mut Vm, exit_evt: EventFd, pci: Option<Arc<Mutex<devices::PciRoot>>>)
                     -> Result<(devices::Bus, Arc<Mutex<devices::Serial>>)> {
         struct NoDevice;
         impl devices::BusDevice for NoDevice {}
@@ -386,7 +478,13 @@ impl arch::LinuxArch for X8664arch {
         io_bus.insert(nul_device.clone(), 0x040, 0x8, false).unwrap(); // ignore pit
         io_bus.insert(nul_device.clone(), 0x0ed, 0x1, false).unwrap(); // most likely this one does nothing
         io_bus.insert(nul_device.clone(), 0x0f0, 0x2, false).unwrap(); // ignore fpu
-        io_bus.insert(nul_device.clone(), 0xcf8, 0x8, false).unwrap(); // ignore pci
+
+        if let Some(pci_root) = pci {
+            io_bus.insert(pci_root, 0xcf8, 0x8, false).unwrap();
+        } else {
+            // ignore pci.
+            io_bus.insert(nul_device.clone(), 0xcf8, 0x8, false).unwrap();
+        }
 
         vm.register_irqfd(&com_evt_1_3, 4).map_err(Error::RegisterIrqfd)?;
         vm.register_irqfd(&com_evt_2_4, 3).map_err(Error::RegisterIrqfd)?;