Remove the device manager and use the new resource allocator

Allow IRQs to be assigned before creating device manager.

For PCI, we need to add devices with interrupts before MMIO setup. Add
the ability to tell the architecture device manager about IRQs that we
have stolen.

There was only one function in device_manager and all of its state is
now delegated to the resource allocator, remove it.

Change-Id: I9afa0e3081a20cb024551ef18ae34fe76a1ef39d
Reviewed-on: https://chromium-review.googlesource.com/1089720
Commit-Ready: Dylan Reid <dgreid@chromium.org>
Tested-by: Dylan Reid <dgreid@chromium.org>
Reviewed-by: Sonny Rao <sonnyrao@chromium.org>

16 files changed, 218 insertions, 344 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 48e93e0..496de72 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5,12 +5,12 @@ dependencies = [
  "arch 0.1.0",
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "data_model 0.1.0",
- "device_manager 0.1.0",
  "devices 0.1.0",
  "kernel_cmdline 0.1.0",
  "kvm 0.1.0",
  "kvm_sys 0.1.0",
  "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "resources 0.1.0",
  "sys_util 0.1.0",
 ]
 
@@ -18,11 +18,11 @@ dependencies = [
 name = "arch"
 version = "0.1.0"
 dependencies = [
- "device_manager 0.1.0",
  "devices 0.1.0",
  "kernel_cmdline 0.1.0",
  "kvm 0.1.0",
  "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "resources 0.1.0",
  "sys_util 0.1.0",
 ]
 
@@ -55,7 +55,6 @@ dependencies = [
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "crosvm_plugin 0.14.0",
  "data_model 0.1.0",
- "device_manager 0.1.0",
  "devices 0.1.0",
  "gpu_buffer 0.1.0",
  "io_jail 0.1.0",
@@ -95,18 +94,6 @@ name = "data_model"
 version = "0.1.0"
 
 [[package]]
-name = "device_manager"
-version = "0.1.0"
-dependencies = [
- "devices 0.1.0",
- "io_jail 0.1.0",
- "kernel_cmdline 0.1.0",
- "kvm 0.1.0",
- "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
- "sys_util 0.1.0",
-]
-
-[[package]]
 name = "devices"
 version = "0.1.0"
 dependencies = [
@@ -370,6 +357,7 @@ dependencies = [
  "data_model 0.1.0",
  "kvm 0.1.0",
  "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "resources 0.1.0",
  "sys_util 0.1.0",
 ]
 
@@ -389,13 +377,13 @@ dependencies = [
  "arch 0.1.0",
  "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "data_model 0.1.0",
- "device_manager 0.1.0",
  "devices 0.1.0",
  "kernel_cmdline 0.1.0",
  "kernel_loader 0.1.0",
  "kvm 0.1.0",
  "kvm_sys 0.1.0",
  "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "resources 0.1.0",
  "sys_util 0.1.0",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index a070622..b669011 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,6 @@ wl-dmabuf = ["devices/wl-dmabuf", "gpu_buffer"]
 [dependencies]
 arch = { path = "arch" }
 devices = { path = "devices" }
-device_manager = { path = "device_manager" }
 gpu_buffer = { path = "gpu_buffer", optional = true }
 io_jail = { path = "io_jail" }
 kvm = { path = "kvm" }
diff --git a/aarch64/Cargo.toml b/aarch64/Cargo.toml
index 7793426..45201ae 100644
--- a/aarch64/Cargo.toml
+++ b/aarch64/Cargo.toml
@@ -6,11 +6,11 @@ authors = ["The Chromium OS Authors"]
 [dependencies]
 arch = { path = "../arch" }
 data_model = { path = "../data_model" }
-device_manager = { path = "../device_manager" }
 devices = { path = "../devices" }
 kernel_cmdline = { path = "../kernel_cmdline" }
 kvm_sys = { path = "../kvm_sys" }
 kvm = { path = "../kvm" }
 sys_util = { path = "../sys_util" }
+resources = { path = "../resources" }
 libc = "*"
 byteorder = "*"
diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs
index 179ebc9..a030d5e 100644
--- a/aarch64/src/lib.rs
+++ b/aarch64/src/lib.rs
@@ -5,13 +5,13 @@
 extern crate arch;
 extern crate byteorder;
 extern crate data_model;
-extern crate device_manager;
 extern crate devices;
 extern crate kernel_cmdline;
 extern crate kvm;
 extern crate kvm_sys;
 extern crate libc;
 extern crate sys_util;
+extern crate resources;
 
 use std::error::{self, Error as Aarch64Error};
 use std::fmt::{self, Display};
@@ -20,7 +20,9 @@ use std::io::stdout;
 use std::sync::{Arc, Mutex};
 use std::ffi::CStr;
 
+use devices::Bus;
 use sys_util::{EventFd, GuestAddress, GuestMemory};
+use resources::{AddressRanges, SystemAllocator};
 use std::os::unix::io::FromRawFd;
 
 use kvm::*;
@@ -202,33 +204,36 @@ impl arch::LinuxArch for AArch64 {
         cmdline
     }
 
-    /// This creates and returns a device_manager object for this vm.
+    /// Returns a system resource allocator.
+    fn get_resource_allocator(mem_size: u64) -> SystemAllocator {
+        let device_addr_start = Self::get_base_dev_pfn(mem_size) * sys_util::pagesize() as u64;
+        AddressRanges::new()
+            .add_device_addresses(device_addr_start, u64::max_value() - device_addr_start)
+            .add_mmio_addresses(AARCH64_MMIO_BASE, 0x10000)
+            .create_allocator(AARCH64_IRQ_BASE).unwrap()
+    }
+
+    /// This adds any early platform devices for this architecture.
     ///
     /// # Arguments
     ///
-    /// * `vm` - the vm object
-    /// * `mem` - A copy of the GuestMemory object for this VM.
-    fn get_device_manager(vm: &mut Vm, mem: GuestMemory) ->
-        Result<device_manager::DeviceManager> {
+    /// * `vm` - The vm to add irqs to.
+    /// * `bus` - The bus to add devices to.
+    fn add_arch_devs(vm: &mut Vm, bus: &mut Bus) -> Result<()> {
         let rtc_evt = EventFd::new()?;
         vm.register_irqfd(&rtc_evt, AARCH64_RTC_IRQ)?;
 
-        let mut dm = device_manager::DeviceManager::new(vm,
-                                                        mem,
-                                                        AARCH64_MMIO_LEN,
-                                                        AARCH64_MMIO_BASE,
-                                                        AARCH64_IRQ_BASE);
         let com_evt_1_3 = EventFd::new()?;
         let serial = Arc::new(Mutex::new(devices::Serial::new_out(
             com_evt_1_3.try_clone()?,
             Box::new(stdout()))));
-        dm.bus.insert(serial.clone(), AARCH64_SERIAL_ADDR,
-                      AARCH64_SERIAL_SIZE).expect("failed to add serial device");
+        bus.insert(serial.clone(), AARCH64_SERIAL_ADDR, AARCH64_SERIAL_SIZE)
+            .expect("failed to add serial device");
 
         let rtc = Arc::new(Mutex::new(devices::pl030::Pl030::new(rtc_evt)));
-        dm.bus.insert(rtc, AARCH64_RTC_ADDR,
-                      AARCH64_RTC_SIZE).expect("failed to add rtc device");
-        Ok(dm)
+        bus.insert(rtc, AARCH64_RTC_ADDR, AARCH64_RTC_SIZE)
+            .expect("failed to add rtc device");
+        Ok(())
     }
 
     /// The creates the interrupt controller device and optionally returns the fd for it.
diff --git a/arch/Cargo.toml b/arch/Cargo.toml
index 2b2aa38..20c5466 100644
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -5,8 +5,8 @@ authors = ["The Chromium OS Authors"]
 
 [dependencies]
 devices = { path = "../devices" }
-device_manager = { path = "../device_manager" }
 kvm = { path = "../kvm" }
 sys_util = { path = "../sys_util" }
+resources = { path = "../resources" }
 kernel_cmdline = { path = "../kernel_cmdline" }
 libc = "*"
diff --git a/arch/src/lib.rs b/arch/src/lib.rs
index 9b781c5..19ea98b 100644
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -3,10 +3,10 @@
 // found in the LICENSE file.
 
 extern crate sys_util;
+extern crate resources;
 extern crate kernel_cmdline;
 extern crate kvm;
 extern crate libc;
-extern crate device_manager;
 extern crate devices;
 
 use std::ffi::CStr;
@@ -14,8 +14,10 @@ use std::fs::File;
 use std::result;
 use std::sync::{Arc, Mutex};
 
+use devices::Bus;
 use kvm::{Kvm, Vm, Vcpu};
 use sys_util::{EventFd, GuestMemory};
+use resources::SystemAllocator;
 
 pub type Result<T> = result::Result<T, Box<std::error::Error>>;
 
@@ -52,6 +54,14 @@ pub trait LinuxArch {
     /// * `mem` - The memory to be used by the guest.
     fn create_vm(kvm: &Kvm, mem: GuestMemory) -> Result<Vm>;
 
+    /// This adds any early platform devices for this architecture.
+    ///
+    /// # Arguments
+    ///
+    /// * `vm` - The vm to add irqs to.
+    /// * `bus` - The bus to add devices to.
+    fn add_arch_devs(_vm: &mut Vm, _bus: &mut Bus) -> Result<()> { Ok(()) }
+
     /// This creates a GuestMemory object for this VM
     ///
     /// * `mem_size` - Desired physical memory size in bytes for this VM
@@ -76,14 +86,8 @@ pub trait LinuxArch {
     /// This returns a minimal kernel command for this architecture.
     fn get_base_linux_cmdline() -> kernel_cmdline::Cmdline;
 
-    /// This creates and returns a device_manager object for this vm.
-    ///
-    /// # Arguments
-    ///
-    /// * `vm` - the vm object
-    /// * `mem` - A copy of the GuestMemory object for this VM.
-    fn get_device_manager(vm: &mut Vm, mem: GuestMemory)
-                          -> Result<device_manager::DeviceManager>;
+    /// Returns a system resource allocator.
+    fn get_resource_allocator(mem_size: u64) -> SystemAllocator;
 
     /// Sets up the IO bus for this platform
     ///
diff --git a/build_test.py b/build_test.py
index f7a4a7e..2dfefe4 100755
--- a/build_test.py
+++ b/build_test.py
@@ -28,7 +28,6 @@ X86_64_TRIPLE = os.getenv('X86_64_TRIPLE', 'x86_64-cros-linux-gnu')
 TEST_MODULES_PARALLEL = [
     'crosvm',
     'data_model',
-    'device_manager',
     'kernel_loader',
     'kvm',
     'kvm_sys',
diff --git a/device_manager/Cargo.toml b/device_manager/Cargo.toml
deleted file mode 100644
index e6b3d15..0000000
--- a/device_manager/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "device_manager"
-version = "0.1.0"
-
-[dependencies]
-devices = { path = "../devices" }
-io_jail = { path = "../io_jail" }
-kvm = { path = "../kvm" }
-sys_util = { path = "../sys_util" }
-kernel_cmdline = { path = "../kernel_cmdline" }
-libc = "*"
-
-[lib]
-path = "src/device_manager.rs"
diff --git a/device_manager/src/device_manager.rs b/device_manager/src/device_manager.rs
deleted file mode 100644
index 8695bba..0000000
--- a/device_manager/src/device_manager.rs
+++ /dev/null
@@ -1,208 +0,0 @@
-// Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-//! Manages jailing and connecting virtio devices to the system bus.
-
-extern crate devices;
-extern crate io_jail;
-extern crate kvm;
-extern crate sys_util;
-extern crate kernel_cmdline;
-
-use std::fmt;
-use std::os::unix::io::{AsRawFd, RawFd};
-use std::sync::{Arc, Mutex};
-
-use io_jail::Minijail;
-use kvm::{Vm, IoeventAddress};
-use sys_util::{GuestMemory, syslog};
-
-/// Errors for device manager.
-#[derive(Debug)]
-pub enum Error {
-    /// Could not create the mmio device to wrap a VirtioDevice.
-    CreateMmioDevice(sys_util::Error),
-    /// Failed to register ioevent with VM.
-    RegisterIoevent(sys_util::Error),
-    /// Failed to register irq eventfd with VM.
-    RegisterIrqfd(sys_util::Error),
-    /// Failed to initialize proxy device for jailed device.
-    ProxyDeviceCreation(devices::ProxyError),
-    /// Appending to kernel command line failed.
-    Cmdline(kernel_cmdline::Error),
-    /// No more IRQs are available.
-    IrqsExhausted,
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            &Error::CreateMmioDevice(ref e) => write!(f, "failed to create mmio device: {:?}", e),
-            &Error::RegisterIoevent(ref e) => {
-                write!(f, "failed to register ioevent to VM: {:?}", e)
-            }
-            &Error::RegisterIrqfd(ref e) => {
-                write!(f, "failed to register irq eventfd to VM: {:?}", e)
-            }
-            &Error::ProxyDeviceCreation(ref e) => write!(f, "failed to create proxy device: {}", e),
-            &Error::Cmdline(ref e) => {
-                write!(f, "unable to add device to kernel command line: {}", e)
-            }
-            &Error::IrqsExhausted => write!(f, "no more IRQs are available"),
-        }
-    }
-}
-
-type Result<T> = ::std::result::Result<T, Error>;
-
-const MAX_IRQ: u32 = 15;
-
-/// Manages the complexities of adding a device.
-pub struct DeviceManager<'a> {
-    pub bus: devices::Bus,
-    vm: &'a mut Vm,
-    guest_mem: GuestMemory,
-    mmio_len: u64,
-    mmio_base: u64,
-    irq: u32,
-}
-
-impl<'a> DeviceManager<'a> {
-    /// Create a new DeviceManager.
-    pub fn new(vm: &mut Vm,
-               guest_mem: GuestMemory,
-               mmio_len: u64,
-               mmio_base: u64,
-               irq_base: u32)
-               -> DeviceManager {
-        DeviceManager {
-            bus: devices::Bus::new(),
-            vm,
-            guest_mem,
-            mmio_len,
-            mmio_base,
-            irq: irq_base,
-        }
-    }
-
-    /// Register a device to be used via MMIO transport.
-    pub fn register_mmio(&mut self,
-                         device: Box<devices::virtio::VirtioDevice>,
-                         jail: Option<Minijail>,
-                         cmdline: &mut kernel_cmdline::Cmdline)
-                         -> Result<()> {
-        if self.irq > MAX_IRQ {
-            return Err(Error::IrqsExhausted);
-        }
-
-        // List of FDs to keep open in the child after it forks.
-        let mut keep_fds: Vec<RawFd> = device.keep_fds();
-        syslog::push_fds(&mut keep_fds);
-
-        let mmio_device = devices::virtio::MmioDevice::new(self.guest_mem.clone(), device)
-            .map_err(Error::CreateMmioDevice)?;
-        for (i, queue_evt) in mmio_device.queue_evts().iter().enumerate() {
-            let io_addr = IoeventAddress::Mmio(self.mmio_base +
-                                               devices::virtio::NOTIFY_REG_OFFSET as u64);
-            self.vm
-                .register_ioevent(&queue_evt, io_addr, i as u32)
-                .map_err(Error::RegisterIoevent)?;
-            keep_fds.push(queue_evt.as_raw_fd());
-        }
-
-        if let Some(interrupt_evt) = mmio_device.interrupt_evt() {
-            self.vm
-                .register_irqfd(&interrupt_evt, self.irq)
-                .map_err(Error::RegisterIrqfd)?;
-            keep_fds.push(interrupt_evt.as_raw_fd());
-        }
-
-        if let Some(jail) = jail {
-            let proxy_dev = devices::ProxyDevice::new(mmio_device, &jail, keep_fds)
-                .map_err(Error::ProxyDeviceCreation)?;
-
-            self.bus
-                .insert(Arc::new(Mutex::new(proxy_dev)),
-                        self.mmio_base,
-                        self.mmio_len)
-                .unwrap();
-        } else {
-            self.bus
-                .insert(Arc::new(Mutex::new(mmio_device)),
-                        self.mmio_base,
-                        self.mmio_len)
-                .unwrap();
-        }
-
-        cmdline
-            .insert("virtio_mmio.device",
-                    &format!("4K@0x{:08x}:{}", self.mmio_base, self.irq))
-            .map_err(Error::Cmdline)?;
-        self.mmio_base += self.mmio_len;
-        self.irq += 1;
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::sync::atomic::AtomicUsize;
-    use std::os::unix::io::RawFd;
-    use sys_util::{EventFd, GuestAddress, GuestMemory};
-    use kvm::*;
-    use DeviceManager;
-    use kernel_cmdline;
-    use devices;
-
-    const QUEUE_SIZES: &'static [u16] = &[64];
-
-    #[allow(dead_code)]
-    struct DummyDevice {
-        dummy: u32,
-    }
-
-    impl devices::virtio::VirtioDevice for DummyDevice {
-        fn keep_fds(&self) -> Vec<RawFd> {
-            Vec::new()
-        }
-
-        fn device_type(&self) -> u32 {
-            0
-        }
-
-        fn queue_max_sizes(&self) -> &[u16] {
-            QUEUE_SIZES
-        }
-
-        #[allow(unused_variables)]
-        #[allow(unused_mut)]
-        fn activate(&mut self,
-                    mem: GuestMemory,
-                    interrupt_evt: EventFd,
-                    status: Arc<AtomicUsize>,
-                    queues: Vec<devices::virtio::Queue>,
-                    mut queue_evts: Vec<EventFd>) {
-        }
-    }
-
-    #[test]
-    #[ignore] // no access to /dev/kvm
-    fn register_device() {
-        let start_addr1 = GuestAddress(0x0);
-        let start_addr2 = GuestAddress(0x1000);
-        let guest_mem = GuestMemory::new(&vec![(start_addr1, 0x1000), (start_addr2, 0x1000)])
-            .unwrap();
-        let mut vm = Vm::new(&Kvm::new().unwrap(), guest_mem.clone()).unwrap();
-        let mut device_manager =
-            DeviceManager::new(&mut vm, guest_mem, 0x1000, 0xd0000000, 5);
-
-        let mut cmdline = kernel_cmdline::Cmdline::new(4096);
-        let dummy_box = Box::new(DummyDevice { dummy: 0 });
-        device_manager
-            .register_mmio(dummy_box, None, &mut cmdline)
-            .unwrap();
-    }
-}
diff --git a/resources/src/system_allocator.rs b/resources/src/system_allocator.rs
index e03f8c2..fec8b41 100644
--- a/resources/src/system_allocator.rs
+++ b/resources/src/system_allocator.rs
@@ -23,7 +23,7 @@ use sys_util::pagesize;
 /// ```
 #[derive(Debug, Eq, PartialEq)]
 pub struct SystemAllocator {
-    io_address_space: AddressAllocator,
+    io_address_space: Option<AddressAllocator>,
     device_address_space: AddressAllocator,
     mmio_address_space: AddressAllocator,
     next_irq: u32,
@@ -41,14 +41,17 @@ impl SystemAllocator {
     /// * `mmio_base` - The starting address of MMIO space.
     /// * `mmio_size` - The size of MMIO space.
     /// * `first_irq` - The first irq number to give out.
-    fn new(io_base: u64, io_size: u64,
-               dev_base: u64, dev_size: u64,
-               mmio_base: u64, mmio_size: u64,
-               first_irq: u32)
-            -> Option<Self> {
+    fn new(io_base: Option<u64>, io_size: Option<u64>,
+           dev_base: u64, dev_size: u64,
+           mmio_base: u64, mmio_size: u64,
+           first_irq: u32) -> Option<Self> {
         let page_size = pagesize() as u64;
         Some(SystemAllocator {
-            io_address_space: AddressAllocator::new(io_base, io_size, Some(0x400))?,
+            io_address_space: if let (Some(b), Some(s)) = (io_base, io_size) {
+                Some(AddressAllocator::new(b, s, Some(0x400))?)
+            } else {
+                None
+            },
             device_address_space: AddressAllocator::new(dev_base, dev_size, Some(page_size))?,
             mmio_address_space: AddressAllocator::new(mmio_base, mmio_size, Some(page_size))?,
             next_irq: first_irq,
@@ -67,7 +70,7 @@ impl SystemAllocator {
 
     /// Reserves a section of `size` bytes of IO address space.
     pub fn allocate_io_addresses(&mut self, size: u64) -> Option<u64> {
-        self.io_address_space.allocate(size)
+        self.io_address_space.as_mut()?.allocate(size)
     }
 
     /// Reserves a section of `size` bytes of device address space.
@@ -122,7 +125,7 @@ impl AddressRanges {
     }
 
     pub fn create_allocator(&self, first_irq: u32) -> Option<SystemAllocator> {
-        SystemAllocator::new(self.io_base?, self.io_size?,
+        SystemAllocator::new(self.io_base, self.io_size,
                              self.device_base?, self.device_size?,
                              self.mmio_base?, self.mmio_size?,
                              first_irq)
diff --git a/src/linux.rs b/src/linux.rs
index 6155870..cc4abed 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -8,7 +8,7 @@ use std::fmt;
 use std::error;
 use std::fs::{File, OpenOptions, remove_file};
 use std::io::{self, stdin};
-use std::os::unix::io::{FromRawFd, RawFd};
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::os::unix::net::UnixDatagram;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -21,7 +21,6 @@ use libc::c_int;
 #[cfg(feature = "wl-dmabuf")]
 use libc::EINVAL;
 
-use device_manager;
 use devices;
 use io_jail::{self, Minijail};
 use kernel_cmdline;
@@ -30,6 +29,7 @@ use net_util::Tap;
 use qcow::{self, QcowFile};
 use sys_util::*;
 use sys_util;
+use resources::SystemAllocator;
 use vhost;
 use vm_control::{VmRequest, GpuMemoryAllocator, GpuMemoryPlaneDesc, GpuMemoryDesc};
 #[cfg(feature = "wl-dmabuf")]
@@ -46,6 +46,7 @@ use x86_64::X8664arch as Arch;
 use aarch64::AArch64 as Arch;
 
 pub enum Error {
+    AddingArchDevs(Box<error::Error>),
     BalloonDeviceNew(devices::virtio::BalloonError),
     BlockDeviceNew(sys_util::Error),
     BlockSignal(sys_util::signal::Error),
@@ -74,13 +75,13 @@ pub enum Error {
     OpenGpuBufferDevice,
     PollContextAdd(sys_util::Error),
     QcowDeviceCreate(qcow::Error),
-    RegisterBalloon(device_manager::Error),
-    RegisterBlock(device_manager::Error),
-    RegisterNet(device_manager::Error),
-    RegisterRng(device_manager::Error),
+    RegisterBalloon(MmioRegisterError),
+    RegisterBlock(MmioRegisterError),
+    RegisterNet(MmioRegisterError),
+    RegisterRng(MmioRegisterError),
     RegisterSignalHandler(sys_util::Error),
-    RegisterVsock(device_manager::Error),
-    RegisterWayland(device_manager::Error),
+    RegisterVsock(MmioRegisterError),
+    RegisterWayland(MmioRegisterError),
     RngDeviceNew(devices::virtio::RngError),
     SettingGidMap(io_jail::Error),
     SettingUidMap(io_jail::Error),
@@ -99,6 +100,7 @@ pub enum Error {
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
+            &Error::AddingArchDevs(ref e) => write!(f, "Failed to add arch devs {:?}", e),
             &Error::BalloonDeviceNew(ref e) => write!(f, "failed to create balloon: {:?}", e),
             &Error::BlockDeviceNew(ref e) => write!(f, "failed to create block device: {:?}", e),
             &Error::BlockSignal(ref e) => write!(f, "failed to block signal: {:?}", e),
@@ -242,16 +244,120 @@ fn create_base_minijail(root: &Path, seccomp_policy: &Path) -> Result<Minijail>
     Ok(j)
 }
 
+/// Errors for device manager.
+#[derive(Debug)]
+pub enum MmioRegisterError {
+    /// Could not create the mmio device to wrap a VirtioDevice.
+    CreateMmioDevice(sys_util::Error),
+    /// Failed to register ioevent with VM.
+    RegisterIoevent(sys_util::Error),
+    /// Failed to register irq eventfd with VM.
+    RegisterIrqfd(sys_util::Error),
+    /// Failed to initialize proxy device for jailed device.
+    ProxyDeviceCreation(devices::ProxyError),
+    /// Appending to kernel command line failed.
+    Cmdline(kernel_cmdline::Error),
+    /// No more IRQs are available.
+    IrqsExhausted,
+    /// No more MMIO space available.
+    AddrsExhausted,
+}
+
+impl fmt::Display for MmioRegisterError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            &MmioRegisterError::CreateMmioDevice(ref e) => write!(f, "failed to create mmio device: {:?}", e),
+            &MmioRegisterError::Cmdline(ref e) => {
+                write!(f, "unable to add device to kernel command line: {}", e)
+            }
+            &MmioRegisterError::RegisterIoevent(ref e) => {
+                write!(f, "failed to register ioevent to VM: {:?}", e)
+            }
+            &MmioRegisterError::RegisterIrqfd(ref e) => {
+                write!(f, "failed to register irq eventfd to VM: {:?}", e)
+            }
+            &MmioRegisterError::ProxyDeviceCreation(ref e) => write!(f, "failed to create proxy device: {}", e),
+            &MmioRegisterError::IrqsExhausted => write!(f, "no more IRQs are available"),
+            &MmioRegisterError::AddrsExhausted => write!(f, "no more addresses are available"),
+        }
+    }
+}
+
+/// Register a device to be used via MMIO transport.
+fn register_mmio(bus: &mut devices::Bus,
+                 vm: &mut Vm,
+                 device: Box<devices::virtio::VirtioDevice>,
+                 jail: Option<Minijail>,
+                 resources: &mut SystemAllocator,
+                 cmdline: &mut kernel_cmdline::Cmdline)
+                 -> std::result::Result<(), MmioRegisterError> {
+    let irq = match resources.allocate_irq() {
+        None => return Err(MmioRegisterError::IrqsExhausted),
+        Some(i) => i,
+    };
+
+    // List of FDs to keep open in the child after it forks.
+    let mut keep_fds: Vec<RawFd> = device.keep_fds();
+    syslog::push_fds(&mut keep_fds);
+
+    let mmio_device = devices::virtio::MmioDevice::new((*vm.get_memory()).clone(),
+    device)
+        .map_err(MmioRegisterError::CreateMmioDevice)?;
+    let mmio_len = 0x1000; // TODO(dgreid) - configurable per arch?
+    let mmio_base = resources.allocate_mmio_addresses(mmio_len)
+        .ok_or(MmioRegisterError::AddrsExhausted)?;
+    for (i, queue_evt) in mmio_device.queue_evts().iter().enumerate() {
+        let io_addr = IoeventAddress::Mmio(mmio_base +
+                                           devices::virtio::NOTIFY_REG_OFFSET as u64);
+        vm.register_ioevent(&queue_evt, io_addr, i as u32)
+          .map_err(MmioRegisterError::RegisterIoevent)?;
+        keep_fds.push(queue_evt.as_raw_fd());
+    }
+
+    if let Some(interrupt_evt) = mmio_device.interrupt_evt() {
+        vm
+            .register_irqfd(&interrupt_evt, irq)
+            .map_err(MmioRegisterError::RegisterIrqfd)?;
+        keep_fds.push(interrupt_evt.as_raw_fd());
+    }
+
+    if let Some(jail) = jail {
+        let proxy_dev = devices::ProxyDevice::new(mmio_device, &jail, keep_fds)
+            .map_err(MmioRegisterError::ProxyDeviceCreation)?;
+
+        bus
+            .insert(Arc::new(Mutex::new(proxy_dev)),
+            mmio_base,
+            mmio_len)
+            .unwrap();
+    } else {
+        bus
+            .insert(Arc::new(Mutex::new(mmio_device)),
+            mmio_base,
+            mmio_len)
+            .unwrap();
+    }
+
+    cmdline
+        .insert("virtio_mmio.device",
+                &format!("4K@0x{:08x}:{}", mmio_base, irq))
+        .map_err(MmioRegisterError::Cmdline)?;
+
+    Ok(())
+}
+
 fn setup_mmio_bus(cfg: &Config,
                   vm: &mut Vm,
                   mem: &GuestMemory,
                   cmdline: &mut kernel_cmdline::Cmdline,
                   control_sockets: &mut Vec<UnlinkUnixDatagram>,
-                  balloon_device_socket: UnixDatagram)
+                  balloon_device_socket: UnixDatagram,
+                  resources: &mut SystemAllocator)
                   -> Result<devices::Bus> {
     static DEFAULT_PIVOT_ROOT: &'static str = "/var/empty";
-    let mut device_manager = Arch::get_device_manager(vm, mem.clone()).
-        map_err(|e| Error::SetupMMIOBus(e))?;
+    let mut bus = devices::Bus::new();
+
+    Arch::add_arch_devs(vm, &mut bus).map_err(Error::AddingArchDevs)?;
 
     // An empty directory for jailed device's pivot root.
     let empty_root_path = Path::new(DEFAULT_PIVOT_ROOT);
@@ -306,8 +412,7 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        device_manager
-            .register_mmio(block_box, jail, cmdline)
+        register_mmio(&mut bus, vm, block_box, jail, resources, cmdline)
             .map_err(Error::RegisterBlock)?;
     }
 
@@ -318,8 +423,7 @@ fn setup_mmio_bus(cfg: &Config,
     } else {
         None
     };
-    device_manager
-        .register_mmio(rng_box, rng_jail, cmdline)
+    register_mmio(&mut bus, vm, rng_box, rng_jail, resources, cmdline)
         .map_err(Error::RegisterRng)?;
 
     let balloon_box = Box::new(devices::virtio::Balloon::new(balloon_device_socket)
@@ -330,7 +434,7 @@ fn setup_mmio_bus(cfg: &Config,
     } else {
         None
     };
-    device_manager.register_mmio(balloon_box, balloon_jail, cmdline)
+    register_mmio(&mut bus, vm, balloon_box, balloon_jail, resources, cmdline)
         .map_err(Error::RegisterBalloon)?;
 
     // We checked above that if the IP is defined, then the netmask is, too.
@@ -348,8 +452,7 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        device_manager
-            .register_mmio(net_box, jail, cmdline)
+        register_mmio(&mut bus, vm, net_box, jail, resources, cmdline)
             .map_err(Error::RegisterNet)?;
     } else if let Some(host_ip) = cfg.host_ip {
         if let Some(netmask) = cfg.netmask {
@@ -377,8 +480,7 @@ fn setup_mmio_bus(cfg: &Config,
                     None
                 };
 
-                device_manager
-                    .register_mmio(net_box, jail, cmdline)
+                register_mmio(&mut bus, vm, net_box, jail, resources, cmdline)
                     .map_err(Error::RegisterNet)?;
             }
         }
@@ -441,8 +543,7 @@ fn setup_mmio_bus(cfg: &Config,
         } else {
             None
         };
-        device_manager
-            .register_mmio(wl_box, jail, cmdline)
+        register_mmio(&mut bus, vm, wl_box, jail, resources, cmdline)
             .map_err(Error::RegisterWayland)?;
     }
 
@@ -458,15 +559,13 @@ fn setup_mmio_bus(cfg: &Config,
             None
         };
 
-        device_manager
-            .register_mmio(vsock_box, jail, cmdline)
+        register_mmio(&mut bus, vm, vsock_box, jail, resources, cmdline)
             .map_err(Error::RegisterVsock)?;
     }
 
-    Ok(device_manager.bus)
+    Ok(bus)
 }
 
-
 fn setup_vcpu(kvm: &Kvm,
               vm: &Vm,
               cpu_id: u32,
@@ -632,9 +731,9 @@ fn create_gpu_memory_allocator() -> Result<Option<Box<GpuMemoryAllocator>>> {
     Ok(None)
 }
 
-fn run_control(vm: &mut Vm,
+fn run_control(mut vm: Vm,
                control_sockets: Vec<UnlinkUnixDatagram>,
-               next_dev_pfn: &mut u64,
+               mut resources: SystemAllocator,
                stdio_serial: Arc<Mutex<devices::Serial>>,
                exit_evt: EventFd,
                sigchld_fd: SignalFd,
@@ -728,8 +827,8 @@ fn run_control(vm: &mut Vm,
                             Ok(request) => {
                                 let mut running = true;
                                 let response =
-                                    request.execute(vm,
-                                                    next_dev_pfn,
+                                    request.execute(&mut vm,
+                                                    &mut resources,
                                                     &mut running,
                                                     &balloon_host_socket,
                                                     gpu_memory_allocator.as_ref().map(|v| {
@@ -816,6 +915,7 @@ pub fn run_config(cfg: Config) -> Result<()> {
     let exit_evt = EventFd::new().map_err(Error::CreateEventFd)?;
 
     let mem_size = cfg.memory.unwrap_or(256) << 20;
+    let mut resources = Arch::get_resource_allocator(mem_size as u64);
     let mem = Arch::setup_memory(mem_size as u64).map_err(|e| Error::CreateGuestMemory(e))?;
     let kvm = Kvm::new().map_err(Error::CreateKvm)?;
     let mut vm = Arch::create_vm(&kvm, mem.clone()).map_err(|e| Error::CreateVm(e))?;
@@ -831,7 +931,6 @@ pub fn run_config(cfg: Config) -> Result<()> {
 
     let irq_chip = Arch::create_irq_chip(&vm).map_err(|e| Error::CreateIrqChip(e))?;
     let mut cmdline = Arch::get_base_linux_cmdline();
-    let mut next_dev_pfn = Arch::get_base_dev_pfn(mem_size as u64);
     let (io_bus, stdio_serial) = Arch::setup_io_bus(&mut vm,
                                                     exit_evt.try_clone().
                                                     map_err(Error::CloneEventFd)?).
@@ -844,7 +943,8 @@ pub fn run_config(cfg: Config) -> Result<()> {
                                   &mem,
                                   &mut cmdline,
                                   &mut control_sockets,
-                                  balloon_device_socket)?;
+                                  balloon_device_socket,
+                                  &mut resources)?;
 
     let gpu_memory_allocator = if cfg.wayland_dmabuf {
         create_gpu_memory_allocator()?
@@ -879,9 +979,9 @@ pub fn run_config(cfg: Config) -> Result<()> {
     }
     vcpu_thread_barrier.wait();
 
-    run_control(&mut vm,
+    run_control(vm,
                 control_sockets,
-                &mut next_dev_pfn,
+                resources,
                 stdio_serial,
                 exit_evt,
                 sigchld_fd,
diff --git a/src/main.rs b/src/main.rs
index 35f4cf0..5ebd517 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -8,7 +8,6 @@ extern crate arch;
 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
 extern crate aarch64;
 extern crate devices;
-extern crate device_manager;
 extern crate libc;
 extern crate io_jail;
 extern crate kvm;
@@ -22,6 +21,7 @@ extern crate net_util;
 extern crate qcow;
 #[macro_use]
 extern crate sys_util;
+extern crate resources;
 extern crate vhost;
 extern crate vm_control;
 extern crate data_model;
diff --git a/vm_control/Cargo.toml b/vm_control/Cargo.toml
index a0d12f5..c2f6e40 100644
--- a/vm_control/Cargo.toml
+++ b/vm_control/Cargo.toml
@@ -9,3 +9,4 @@ data_model = { path = "../data_model" }
 kvm = { path = "../kvm" }
 libc = "*"
 sys_util = { path = "../sys_util" }
+resources = { path = "../resources" }
diff --git a/vm_control/src/lib.rs b/vm_control/src/lib.rs
index f13faef..f90520d 100644
--- a/vm_control/src/lib.rs
+++ b/vm_control/src/lib.rs
@@ -15,6 +15,7 @@ extern crate data_model;
 extern crate kvm;
 extern crate libc;
 extern crate sys_util;
+extern crate resources;
 
 use std::fs::File;
 use std::io::{Seek, SeekFrom};
@@ -27,6 +28,7 @@ use libc::{ERANGE, EINVAL, ENODEV};
 use byteorder::{LittleEndian, WriteBytesExt};
 use data_model::{DataInit, Le32, Le64, VolatileMemory};
 use sys_util::{EventFd, Result, Error as SysError, MmapError, MemoryMapping, Scm, GuestAddress};
+use resources::SystemAllocator;
 use kvm::{IoeventAddress, Vm};
 
 #[derive(Debug, PartialEq)]
@@ -108,26 +110,24 @@ struct VmRequestStruct {
 // Safe because it only has data and has no implicit padding.
 unsafe impl DataInit for VmRequestStruct {}
 
-fn register_memory(vm: &mut Vm, next_mem_pfn: &mut u64, fd: &AsRawFd, size: usize) -> Result<(u64, u32)> {
+fn register_memory(vm: &mut Vm, allocator: &mut SystemAllocator,
+                   fd: &AsRawFd, size: usize) -> Result<(u64, u32)> {
     let mmap = match MemoryMapping::from_fd(fd, size) {
         Ok(v) => v,
         Err(MmapError::SystemCallFailed(e)) => return Err(e),
         _ => return Err(SysError::new(EINVAL)),
     };
-    let pfn = *next_mem_pfn;
+    let addr = match allocator.allocate_device_addresses(size as u64) {
+        Some(a) => a,
+        None => return Err(SysError::new(EINVAL)),
+    };
     let slot =
-        match vm.add_device_memory(GuestAddress(pfn << 12), mmap, false, false) {
+        match vm.add_device_memory(GuestAddress(addr), mmap, false, false) {
             Ok(v) => v,
             Err(e) => return Err(e),
         };
-    // TODO(zachr): Use a smarter allocation strategy. The current strategy is just
-    // bumping this pointer, meaning the remove operation does not free any address
-    // space. Given enough allocations, device memory may run out of address space and
-    // collide with guest memory or MMIO address space. There is currently nothing in
-    // place to limit the amount of address space used by device memory.
-    *next_mem_pfn += (((size + 0x7ff) >> 12) + 1) as u64;
-
-    Ok((pfn, slot))
+
+    Ok((addr >> 12, slot))
 }
 
 /// Struct that describes the offset and stride of a plane located in GPU memory.
@@ -240,14 +240,13 @@ impl VmRequest {
     ///
     /// # Arguments
     /// * `vm` - The `Vm` to perform the request on.
-    /// * `next_mem_pfn` - In/out argument for the page frame number to put the next chunk of device
-    /// memory into.
+    /// * `allocator` - Used to allocate addresses.
     /// * `running` - Out argument that is set to false if the request was to stop running the VM.
     ///
     /// This does not return a result, instead encapsulating the success or failure in a
     /// `VmResponse` with the intended purpose of sending the response back over the  socket that
     /// received this `VmRequest`.
-    pub fn execute(&self, vm: &mut Vm, next_mem_pfn: &mut u64, running: &mut bool,
+    pub fn execute(&self, vm: &mut Vm, sys_allocator: &mut SystemAllocator, running: &mut bool,
                    balloon_host_socket: &UnixDatagram,
                    gpu_memory_allocator: Option<&GpuMemoryAllocator>) -> VmResponse {
         *running = true;
@@ -269,7 +268,7 @@ impl VmRequest {
                 }
             }
             &VmRequest::RegisterMemory(ref fd, size) => {
-                match register_memory(vm, next_mem_pfn, fd, size) {
+                match register_memory(vm, sys_allocator, fd, size) {
                     Ok((pfn, slot)) => VmResponse::RegisterMemory { pfn, slot },
                     Err(e) => VmResponse::Err(e),
                 }
@@ -290,11 +289,11 @@ impl VmRequest {
                 }
             }
             &VmRequest::AllocateAndRegisterGpuMemory {width, height, format} => {
-                let allocator = match gpu_memory_allocator {
+                let gpu_allocator = match gpu_memory_allocator {
                     Some(v) => v,
                     None => return VmResponse::Err(SysError::new(ENODEV)),
                 };
-                let (mut fd, desc) = match allocator.allocate(width, height, format) {
+                let (mut fd, desc) = match gpu_allocator.allocate(width, height, format) {
                     Ok(v) => v,
                     Err(e) => return VmResponse::Err(e),
                 };
@@ -304,7 +303,7 @@ impl VmRequest {
                     Ok(v) => v,
                     Err(e) => return VmResponse::Err(SysError::from(e)),
                 };
-                match register_memory(vm, next_mem_pfn, &fd, size as usize) {
+                match register_memory(vm, sys_allocator, &fd, size as usize) {
                     Ok((pfn, slot)) => VmResponse::AllocateAndRegisterGpuMemory {
                         fd: MaybeOwnedFd::Owned(fd),
                         pfn,
diff --git a/x86_64/Cargo.toml b/x86_64/Cargo.toml
index 3a88f98..5baf336 100644
--- a/x86_64/Cargo.toml
+++ b/x86_64/Cargo.toml
@@ -7,10 +7,10 @@ authors = ["The Chromium OS Authors"]
 arch = { path = "../arch" }
 data_model = { path = "../data_model" }
 devices = { path = "../devices" }
-device_manager = { path = "../device_manager" }
 kvm_sys = { path = "../kvm_sys" }
 kvm = { path = "../kvm" }
 sys_util = { path = "../sys_util" }
+resources = { path = "../resources" }
 kernel_cmdline = { path = "../kernel_cmdline" }
 kernel_loader = { path = "../kernel_loader" }
 libc = "*"
diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs
index 1fcaed0..1e13e77 100644
--- a/x86_64/src/lib.rs
+++ b/x86_64/src/lib.rs
@@ -6,11 +6,11 @@ extern crate arch;
 extern crate byteorder;
 extern crate data_model;
 extern crate devices;
-extern crate device_manager;
 extern crate kvm;
 extern crate kvm_sys;
 extern crate libc;
 extern crate sys_util;
+extern crate resources;
 extern crate kernel_cmdline;
 extern crate kernel_loader;
 
@@ -70,6 +70,7 @@ use std::io::stdout;
 use bootparam::boot_params;
 use bootparam::E820_RAM;
 use sys_util::{EventFd, GuestAddress, GuestMemory};
+use resources::{AddressRanges, SystemAllocator};
 use kvm::*;
 
 #[derive(Debug)]
@@ -132,6 +133,7 @@ const ZERO_PAGE_OFFSET: u64 = 0x7000;
 const KERNEL_START_OFFSET: u64 = 0x200000;
 const CMDLINE_OFFSET: u64 = 0x20000;
 const CMDLINE_MAX_SIZE: u64 = KERNEL_START_OFFSET - CMDLINE_OFFSET;
+const X86_64_IRQ_BASE: u32 = 5;
 
 fn configure_system(guest_mem: &GuestMemory,
                     kernel_addr: GuestAddress,
@@ -314,19 +316,15 @@ impl arch::LinuxArch for X8664arch {
         cmdline
     }
 
-    /// This creates and returns a device_manager object for this vm.
-    ///
-    /// # Arguments
-    ///
-    /// * `vm` - the vm object
-    /// * `mem` - A copy of the GuestMemory object for this VM.
-    fn get_device_manager(vm: &mut Vm, mem: GuestMemory) ->
-        Result<device_manager::DeviceManager> {
-        const MMIO_BASE: u64 = 0xd0000000;
-        const MMIO_LEN: u64 = 0x1000;
-        const IRQ_BASE: u32 = 5;
-
-        Ok(device_manager::DeviceManager::new(vm, mem, MMIO_LEN, MMIO_BASE, IRQ_BASE))
+    /// Returns a system resource allocator.
+    fn get_resource_allocator(mem_size: u64) -> SystemAllocator {
+        const MMIO_BASE: u64 = 0xe0000000;
+        let device_addr_start = Self::get_base_dev_pfn(mem_size) * sys_util::pagesize() as u64;
+        AddressRanges::new()
+           .add_io_addresses(0xc000, 0x10000)
+           .add_mmio_addresses(MMIO_BASE, 0x10000)
+           .add_device_addresses(device_addr_start, u64::max_value() - device_addr_start)
+           .create_allocator(X86_64_IRQ_BASE).unwrap()
     }
 
     /// Sets up the IO bus for this platform