crosvm: refactor and expand vm control socket IPC

This CL adds VM request capabilities to the control socket. These
requests include the basic exit as well as the essential ioeventfd and
irqfd requests. For virtio wayland, the register/unregister device
memory request was added.

TEST=cargo test
BUG=chromium:738638

Change-Id: I0cbf62d85a299cf454bcf6924a4e1d52d5b7183f
Reviewed-on: https://chromium-review.googlesource.com/602593
Commit-Ready: Zach Reizner <zachr@chromium.org>
Tested-by: Zach Reizner <zachr@chromium.org>
Reviewed-by: Dylan Reid <dgreid@chromium.org>

8 files changed, 552 insertions, 257 deletions

diff --git a/Cargo.toml b/Cargo.toml
index 1c1e60b..9ee038f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,6 +20,7 @@ net_sys = { path = "net_sys" }
 net_util = { path = "net_util" }
 vhost = { path = "vhost" }
 virtio_sys = { path = "virtio_sys" }
+data_model = { path = "data_model" }
 
 [dependencies.clap]
 version = "*"
diff --git a/data_model/src/endian.rs b/data_model/src/endian.rs
index d1a25a9..97d1b72 100644
--- a/data_model/src/endian.rs
+++ b/data_model/src/endian.rs
@@ -37,7 +37,7 @@ macro_rules! endian_type {
         /// An unsigned integer type of with an explicit endianness.
         ///
         /// See module level documentation for examples.
-        #[derive(Copy, Clone, Eq, PartialEq, Debug)]
+        #[derive(Copy, Clone, Eq, PartialEq, Debug, Default)]
         pub struct $new_type($old_type);
 
         impl $new_type {
diff --git a/kvm/src/lib.rs b/kvm/src/lib.rs
index 084365c..4f2d3a6 100644
--- a/kvm/src/lib.rs
+++ b/kvm/src/lib.rs
@@ -135,6 +135,7 @@ impl AsRawFd for Kvm {
 }
 
 /// An address either in programmable I/O space or in memory mapped I/O space.
+#[derive(Copy, Clone)]
 pub enum IoeventAddress {
     Pio(u64),
     Mmio(u64),
diff --git a/src/control_socket.rs b/src/control_socket.rs
deleted file mode 100644
index 5e5f9a3..0000000
--- a/src/control_socket.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-//! Helper for sending and receiving control socket commands
-
-use std::fs::remove_file;
-use std::io::Result;
-use std::os::unix::io::{AsRawFd, RawFd};
-use std::os::unix::net::UnixDatagram;
-use std::path::Path;
-
-use libc::getpid;
-
-use sys_util::Pollable;
-
-#[derive(Debug, PartialEq)]
-/// The set of commands that can be sent over a control socket.
-pub enum Command {
-    /// A command that could not be identified.
-    Unknown,
-    /// Stop the process in an orderly fashion.
-    Stop,
-}
-
-impl Command {
-    fn decode(data: &[u8]) -> Command {
-        if data.is_empty() {
-            return Command::Unknown;
-        }
-        if data[0] == b's' {
-            return Command::Stop;
-        }
-        Command::Unknown
-    }
-
-    fn encode(&self, data: &mut [u8]) {
-        if data.is_empty() {
-            return;
-        }
-        match *self {
-            Command::Unknown => data[0] = 0,
-            Command::Stop => data[0] = b's',
-        }
-    }
-}
-
-/// The receiving side of a control socket.
-pub struct ControlSocketRecv {
-    s: UnixDatagram,
-}
-
-impl ControlSocketRecv {
-    /// Binds a new control socket at `path`.
-    pub fn new<P: AsRef<Path>>(path: P) -> Result<ControlSocketRecv> {
-        let path = path.as_ref();
-        let control_socket = if path.is_dir() {
-            // Getting the pid is always safe and never fails.
-            let pid = unsafe { getpid() };
-            UnixDatagram::bind(path.join(format!("crosvm-{}.sock", pid)))?
-        } else {
-            UnixDatagram::bind(path)?
-        };
-        Ok(ControlSocketRecv { s: control_socket })
-    }
-
-    /// Use the already connected `socket` as a control socket.
-    pub fn with_socket(socket: UnixDatagram) -> ControlSocketRecv {
-        ControlSocketRecv { s: socket }
-    }
-
-    /// Receives a command on this control socket.
-    ///
-    /// Note that this will block if there is no command waiting.
-    pub fn recv(&self) -> Result<Command> {
-        let mut buf = [0; 32];
-        self.s.recv(&mut buf)?;
-        Ok(Command::decode(&buf))
-    }
-}
-
-
-// Safe because we return a genuine pollable fd that never changes and shares our lifetime.
-unsafe impl Pollable for ControlSocketRecv {
-    fn pollable_fd(&self) -> RawFd {
-        self.s.as_raw_fd()
-    }
-}
-
-impl Drop for ControlSocketRecv {
-    fn drop(&mut self) {
-        if let Ok(addr) = self.s.local_addr() {
-            if let Some(path) = addr.as_pathname() {
-                if let Err(e) = remove_file(path) {
-                    println!("failed to remove control socket file: {:?}", e);
-                }
-            }
-        }
-    }
-}
-
-/// The sending side of the control socket.
-pub struct ControlSocketSend {
-    s: UnixDatagram,
-}
-
-impl ControlSocketSend {
-    /// Connect to a control socket at `path`.
-    pub fn new<P: AsRef<Path>>(path: P) -> Result<ControlSocketSend> {
-        let s = UnixDatagram::unbound()?;
-        s.connect(path)?;
-        Ok(ControlSocketSend { s: s })
-    }
-
-    /// Use the already connected `socket` as a control socket.
-    pub fn with_socket(socket: UnixDatagram) -> ControlSocketSend {
-        ControlSocketSend { s: socket }
-    }
-
-    /// Send a command to this control socket.
-    pub fn send(&self, cmd: &Command) -> Result<()> {
-        let mut buf = [0; 32];
-        cmd.encode(&mut buf);
-        self.s.send(&buf).map(|_| ())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::env::temp_dir;
-    use sys_util::Poller;
-
-    #[test]
-    fn send_recv() {
-        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
-        let send = ControlSocketSend::with_socket(s1);
-        let recv = ControlSocketRecv::with_socket(s2);
-        send.send(&Command::Stop)
-            .expect("failed to send stop command");
-        let cmd = recv.recv().expect("failed to recv stop command");
-        assert_eq!(cmd, Command::Stop);
-    }
-
-    #[test]
-    fn poll_recv() {
-        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
-        let send = ControlSocketSend::with_socket(s1);
-        let recv = ControlSocketRecv::with_socket(s2);
-        send.send(&Command::Stop)
-            .expect("failed to send stop command");
-
-        let mut poller = Poller::new(1);
-        let polled = poller
-            .poll(&[(0, &recv)])
-            .expect("failed to poll recv socket");
-        assert_eq!(polled, [0].as_ref());
-
-        let cmd = recv.recv().expect("failed to recv stop command");
-        assert_eq!(cmd, Command::Stop);
-    }
-
-    #[test]
-    fn unknown() {
-        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
-        let send = ControlSocketSend::with_socket(s1);
-        let recv = ControlSocketRecv::with_socket(s2);
-        send.send(&Command::Unknown)
-            .expect("failed to send command");
-        let cmd = recv.recv().expect("failed to recv command");
-        assert_eq!(cmd, Command::Unknown);
-    }
-
-    #[test]
-    fn recv_socket_remove_on_drop() {
-        let pid = unsafe { getpid() };
-        let mut temp_file = temp_dir();
-        temp_file.push(format!("control_socket_drop_recv_socket_{}", pid));
-        assert!(!temp_file.exists());
-        {
-            let recv = ControlSocketRecv::new(temp_file.to_str().unwrap());
-            assert!(temp_file.exists());
-        }
-        assert!(!temp_file.exists());
-    }
-}
diff --git a/src/device_manager.rs b/src/device_manager.rs
index c122d8b..742346a 100644
--- a/src/device_manager.rs
+++ b/src/device_manager.rs
@@ -11,11 +11,12 @@ use std::sync::{Arc, Mutex};
 
 use io_jail::Minijail;
 use kvm::IoeventAddress;
-use sys_util::{EventFd, GuestMemory};
+use sys_util::GuestMemory;
+use sys_util;
 
 use hw;
 use kernel_cmdline;
-use sys_util;
+use vm_control::VmRequest;
 
 /// Errors for device manager.
 #[derive(Debug)]
@@ -53,13 +54,6 @@ impl fmt::Display for Error {
 
 type Result<T> = ::std::result::Result<T, Error>;
 
-/// Represents a request to register an ioeventfd or irqfd, which will be
-/// processed once the VM starts up.
-pub enum VmRequest {
-    RegisterIoevent(EventFd, IoeventAddress, u32),
-    RegisterIrqfd(EventFd, u32),
-}
-
 const MAX_IRQ: u32 = 15;
 
 /// Manages the complexities of adding a device.
@@ -158,7 +152,7 @@ mod tests {
     use super::*;
     use std::sync::atomic::AtomicUsize;
     use std::os::unix::io::RawFd;
-    use sys_util::{GuestAddress, GuestMemory};
+    use sys_util::{EventFd, GuestAddress, GuestMemory};
     use device_manager;
     use kernel_cmdline;
     use hw;
diff --git a/src/main.rs b/src/main.rs
index 45138f9..9233629 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -16,35 +16,40 @@ extern crate net_sys;
 extern crate net_util;
 extern crate vhost;
 extern crate virtio_sys;
+extern crate data_model;
 
-use std::ffi::{CString, CStr};
+pub mod hw;
+pub mod kernel_cmdline;
+pub mod vm_control;
+pub mod device_manager;
+
+use std::ffi::{OsString, CString, CStr};
 use std::fmt;
 use std::fs::File;
 use std::fs::OpenOptions;
 use std::io::{stdin, stdout};
 use std::net;
+use std::os::unix::net::UnixDatagram;
 use std::path::{Path, PathBuf};
 use std::ptr;
 use std::string::String;
-use std::sync::{Arc, Mutex, Barrier};
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, Barrier};
 use std::thread::{spawn, sleep, JoinHandle};
 use std::time::Duration;
 
+use libc::getpid;
+
 use clap::{Arg, App, SubCommand};
 
-use device_manager::{DeviceManager, VmRequest};
+
 use io_jail::Minijail;
 use kvm::*;
-use sys_util::{GuestAddress, GuestMemory, EventFd, TempDir, Terminal, Poller, Pollable,
+use sys_util::{GuestAddress, GuestMemory, EventFd, TempDir, Terminal, Poller, Pollable, Scm,
                register_signal_handler, Killable, SignalFd, syslog};
 
-pub mod hw;
-pub mod kernel_cmdline;
-pub mod control_socket;
-pub mod device_manager;
-
-use control_socket::*;
+use device_manager::*;
+use vm_control::{VmRequest, VmResponse};
 
 enum Error {
     Socket(std::io::Error),
@@ -159,6 +164,7 @@ struct Config<'a> {
 const KERNEL_START_OFFSET: usize = 0x200000;
 const CMDLINE_OFFSET: usize = 0x20000;
 const CMDLINE_MAX_SIZE: usize = KERNEL_START_OFFSET - CMDLINE_OFFSET;
+const BASE_DEV_MEMORY_PFN: u64 = 1u64 << 26;
 
 fn create_base_minijail(root: &Path, seccomp_policy: &Path) -> Result<Minijail> {
     // All child jails run in a new user namespace without any users mapped,
@@ -231,12 +237,19 @@ fn run_config(cfg: Config) -> Result<()> {
         return Err(Error::NetMissingConfig);
     }
 
-    let socket = if let Some(ref socket_path) = cfg.socket_path {
-        Some(ControlSocketRecv::new(socket_path)
-                 .map_err(|e| Error::Socket(e))?)
-    } else {
-        None
-    };
+    let mut control_sockets = Vec::new();
+    if let Some(ref path) = cfg.socket_path {
+        let path = Path::new(path);
+        let control_socket = if path.is_dir() {
+                // Getting the pid is always safe and never fails.
+                let pid = unsafe { getpid() };
+                UnixDatagram::bind(path.join(format!("crosvm-{}.sock", pid)))
+            } else {
+                UnixDatagram::bind(path)
+            }
+            .map_err(|e| Error::Socket(e))?;
+        control_sockets.push(control_socket);
+    }
 
     let mem_size = cfg.memory.unwrap_or(256) << 20;
     let guest_mem =
@@ -328,7 +341,7 @@ fn run_config(cfg: Config) -> Result<()> {
             cfg.vcpu_count.unwrap_or(1),
             guest_mem,
             &device_manager.bus,
-            socket,
+            control_sockets,
             cfg.warn_unknown_ports)
 }
 
@@ -338,7 +351,7 @@ fn run_kvm(requests: Vec<VmRequest>,
            vcpu_count: u32,
            guest_mem: GuestMemory,
            mmio_bus: &hw::Bus,
-           control_socket: Option<ControlSocketRecv>,
+           control_sockets: Vec<UnixDatagram>,
            warn_unknown_ports: bool)
            -> Result<()> {
     let kvm = Kvm::new().map_err(Error::Kvm)?;
@@ -346,21 +359,20 @@ fn run_kvm(requests: Vec<VmRequest>,
     let kernel_start_addr = GuestAddress(KERNEL_START_OFFSET);
     let cmdline_addr = GuestAddress(CMDLINE_OFFSET);
 
-    let vm = Vm::new(&kvm, guest_mem).map_err(Error::Vm)?;
+    let mut vm = Vm::new(&kvm, guest_mem).map_err(Error::Vm)?;
     vm.set_tss_addr(tss_addr).expect("set tss addr failed");
     vm.create_pit().expect("create pit failed");
     vm.create_irq_chip().expect("create irq chip failed");
 
+    let mut next_dev_pfn = BASE_DEV_MEMORY_PFN;
     for request in requests {
-        match request {
-            VmRequest::RegisterIoevent(evt, addr, datamatch) => {
-                vm.register_ioevent(&evt, addr, datamatch)
-                    .map_err(Error::RegisterIoevent)?
-            }
-            VmRequest::RegisterIrqfd(evt, irq) => {
-                vm.register_irqfd(&evt, irq)
-                    .map_err(Error::RegisterIrqfd)?
-            }
+        let mut running = false;
+        if let VmResponse::Err(e) = request.execute(&mut vm, &mut next_dev_pfn, &mut running) {
+            return Err(Error::Vm(e));
+        }
+        if !running {
+            println!("configuration requested exit");
+            return Ok(());
         }
     }
 
@@ -518,7 +530,9 @@ fn run_kvm(requests: Vec<VmRequest>,
 
     vcpu_thread_barrier.wait();
 
-    run_control(control_socket,
+    run_control(vm,
+                control_sockets,
+                next_dev_pfn,
                 stdio_serial,
                 exit_evt,
                 sigchld_fd,
@@ -526,17 +540,21 @@ fn run_kvm(requests: Vec<VmRequest>,
                 vcpu_handles)
 }
 
-fn run_control(control_socket: Option<ControlSocketRecv>,
+fn run_control(mut vm: Vm,
+               control_sockets: Vec<UnixDatagram>,
+               mut next_dev_pfn: u64,
                stdio_serial: Arc<Mutex<hw::Serial>>,
                exit_evt: EventFd,
                sigchld_fd: SignalFd,
                kill_signaled: Arc<AtomicBool>,
                vcpu_handles: Vec<JoinHandle<()>>)
                -> Result<()> {
-    const EXIT: u32 = 1;
-    const STDIN: u32 = 2;
-    const CONTROL: u32 = 3;
-    const CHILD_SIGNAL: u32 = 4;
+    const MAX_VM_FD_RECV: usize = 1;
+
+    const EXIT: u32 = 0;
+    const STDIN: u32 = 1;
+    const CHILD_SIGNAL: u32 = 2;
+    const VM_BASE: u32 = 3;
 
     let stdin_handle = stdin();
     let stdin_lock = stdin_handle.lock();
@@ -547,15 +565,16 @@ fn run_control(control_socket: Option<ControlSocketRecv>,
     let mut pollables = Vec::new();
     pollables.push((EXIT, &exit_evt as &Pollable));
     pollables.push((STDIN, &stdin_lock as &Pollable));
-    if let Some(socket) = control_socket.as_ref() {
-        pollables.push((CONTROL, socket as &Pollable));
-    }
     pollables.push((CHILD_SIGNAL, &sigchld_fd as &Pollable));
+    for (i, socket) in control_sockets.iter().enumerate() {
+        pollables.push((VM_BASE + i as u32, socket as &Pollable));
+    }
 
-    let mut poller = Poller::new(4);
+    let mut poller = Poller::new(pollables.len());
+    let mut scm = Scm::new(MAX_VM_FD_RECV);
 
     'poll: loop {
-        let poll_res = {
+        let tokens = {
             match poller.poll(&pollables[..]) {
                 Ok(v) => v,
                 Err(e) => {
@@ -564,8 +583,8 @@ fn run_control(control_socket: Option<ControlSocketRecv>,
                 }
             }
         };
-        for i in poll_res {
-            match *i {
+        for &token in tokens {
+            match token {
                 EXIT => {
                     println!("vcpu requested shutdown");
                     break 'poll;
@@ -581,17 +600,6 @@ fn run_control(control_socket: Option<ControlSocketRecv>,
                             .expect("failed to queue bytes into serial port");
                     }
                 }
-                CONTROL if control_socket.is_some() => {
-                    if let Some(socket) = control_socket.as_ref() {
-                        match socket.recv().unwrap() {
-                            Command::Stop => {
-                                println!("control socket requested shutdown");
-                                break 'poll;
-                            }
-                            _ => {}
-                        }
-                    }
-                }
                 CHILD_SIGNAL => {
                     // Print all available siginfo structs, then exit the loop.
                     loop {
@@ -606,6 +614,24 @@ fn run_control(control_socket: Option<ControlSocketRecv>,
                         break 'poll;
                     }
                 }
+                t if t >= VM_BASE && t < VM_BASE + (control_sockets.len() as u32) => {
+                    let socket = &control_sockets[(t - VM_BASE) as usize];
+                    match VmRequest::recv(&mut scm, socket) {
+                        Ok(request) => {
+                            let mut running = true;
+                            let response =
+                                request.execute(&mut vm, &mut next_dev_pfn, &mut running);
+                            if let Err(e) = response.send(&mut scm, socket) {
+                                println!("failed to send VmResponse: {:?}", e);
+                            }
+                            if !running {
+                                println!("control socket requested exit");
+                                break 'poll;
+                            }
+                        }
+                        Err(e) => println!("failed to recv VmRequest: {:?}", e),
+                    }
+                }
                 _ => {}
             }
         }
@@ -718,15 +744,20 @@ fn main() {
 
     match matches.subcommand() {
         ("stop", Some(matches)) => {
+            let mut scm = Scm::new(1);
             for socket_path in matches.values_of("socket").unwrap() {
-                let res = match ControlSocketSend::new(socket_path) {
-                    Ok(s) => s.send(&Command::Stop),
-                    Err(e) => Err(e),
-                };
-                if let Err(e) = res {
-                    println!("failed to send stop command to socket at '{}': {}",
-                             socket_path,
-                             e);
+                match UnixDatagram::unbound().and_then(|s| {
+                                                           s.connect(socket_path)?;
+                                                           Ok(s)
+                                                       }) {
+                    Ok(s) => {
+                        if let Err(e) = VmRequest::Exit.send(&mut scm, &s) {
+                            println!("failed to send stop request to socket at '{}': {:?}",
+                                     socket_path,
+                                     e);
+                        }
+                    }
+                    Err(e) => println!("failed to connect to socket at '{}': {}", socket_path, e),
                 }
             }
         }
diff --git a/src/vm_control.rs b/src/vm_control.rs
new file mode 100644
index 0000000..8937be2
--- /dev/null
+++ b/src/vm_control.rs
@@ -0,0 +1,453 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+//! Handles IPC for controlling the main VM process.
+//!
+//! The VM Control IPC protocol is synchronous, meaning that each `VmRequest` sent over a connection
+//! will receive a `VmResponse` for that request next time data is received over that connection.
+//!
+//! The wire message format is a little-endian C-struct of fixed size, along with a file descriptor
+//! if the request type expects one.
+
+use std::fs::File;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::net::UnixDatagram;
+use std::result;
+
+use libc::{ERANGE, EINVAL};
+
+use data_model::{DataInit, Le32, Le64, VolatileMemory};
+use sys_util::{EventFd, Error as SysError, MmapError, MemoryMapping, Scm, GuestAddress};
+use kvm::{IoeventAddress, Vm};
+
+#[derive(Debug, PartialEq)]
+/// An error during a request or response transaction.
+pub enum VmControlError {
+    /// Error while sending a request or response.
+    Send(SysError),
+    /// Error while receiving a request or response.
+    Recv(SysError),
+    /// The type of a received request or response is unknown.
+    InvalidType,
+    /// There was not the expected amount of data when receiving a request or response. The inner
+    /// value is how much data was read.
+    BadSize(usize),
+    /// There was no associated file descriptor received for a request that expected it.
+    ExpectFd,
+}
+
+pub type VmControlResult<T> = result::Result<T, VmControlError>;
+
+/// A file descriptor either borrowed or owned by this.
+pub enum MaybeOwnedFd {
+    /// Owned by this enum variant, and will be destructed automatically if not moved out.
+    Owned(File),
+    /// A file descriptor borrwed by this enum.
+    Borrowed(RawFd),
+}
+
+impl AsRawFd for MaybeOwnedFd {
+    fn as_raw_fd(&self) -> RawFd {
+        match self {
+            &MaybeOwnedFd::Owned(ref f) => f.as_raw_fd(),
+            &MaybeOwnedFd::Borrowed(fd) => fd,
+        }
+    }
+}
+
+/// A request to the main process to perform some operation on the VM.
+///
+/// Unless otherwise noted, each request should expect a `VmResponse::Ok` to be received on success.
+pub enum VmRequest {
+    /// Break the VM's run loop and exit.
+    Exit,
+    /// Register the given ioevent address along with given datamatch to trigger the `EventFd`.
+    RegisterIoevent(EventFd, IoeventAddress, u32),
+    /// Register the given IRQ number to be triggered when the `EventFd` is triggered.
+    RegisterIrqfd(EventFd, u32),
+    /// Register shared memory represented by the given fd into guest address space. The response
+    /// variant is `VmResponse::RegisterMemory`.
+    RegisterMemory(MaybeOwnedFd, usize),
+    /// Unregister the given memory slot that was previously registereed with `RegisterMemory`.
+    UnregisterMemory(u32),
+}
+
+const VM_REQUEST_TYPE_EXIT: u32 = 1;
+const VM_REQUEST_TYPE_REGISTER_MEMORY: u32 = 2;
+const VM_REQUEST_TYPE_UNREGISTER_MEMORY: u32 = 3;
+const VM_REQUEST_SIZE: usize = 16;
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct VmRequestStruct {
+    type_: Le32,
+    slot: Le32,
+    size: Le64,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl DataInit for VmRequestStruct {}
+
+impl VmRequest {
+    /// Receive a `VmRequest` from the given socket.
+    ///
+    /// A `VmResponse` should be sent out over the given socket before another request is received.
+    pub fn recv(scm: &mut Scm, s: &UnixDatagram) -> VmControlResult<VmRequest> {
+        let mut buf = [0; VM_REQUEST_SIZE];
+        let mut fds = Vec::new();
+        let read = scm.recv(s, &mut [&mut buf], &mut fds)
+            .map_err(|e| VmControlError::Recv(e))?;
+        if read != VM_REQUEST_SIZE {
+            return Err(VmControlError::BadSize(read));
+        }
+        // The unwrap() will never fail because it's referencing a buf statically sized to be large
+        // enough for a VmRequestStruct.
+        let req: VmRequestStruct = buf.as_mut().get_ref(0).unwrap().load();
+
+        match req.type_.into() {
+            VM_REQUEST_TYPE_EXIT => Ok(VmRequest::Exit),
+            VM_REQUEST_TYPE_REGISTER_MEMORY => {
+                let fd = fds.pop().ok_or(VmControlError::ExpectFd)?;
+                Ok(VmRequest::RegisterMemory(MaybeOwnedFd::Owned(fd),
+                                             req.size.to_native() as usize))
+            }
+            VM_REQUEST_TYPE_UNREGISTER_MEMORY => Ok(VmRequest::UnregisterMemory(req.slot.into())),
+            _ => Err(VmControlError::InvalidType),
+        }
+    }
+
+    /// Send a `VmRequest` over the given socket.
+    ///
+    /// After this request is a sent, a `VmResponse` should be received before sending another
+    /// request.
+    pub fn send(&self, scm: &mut Scm, s: &UnixDatagram) -> VmControlResult<()> {
+        let mut req = VmRequestStruct::default();
+        let mut fd_buf = [0; 1];
+        let mut fd_len = 0;
+        match self {
+            &VmRequest::Exit => req.type_ = Le32::from(VM_REQUEST_TYPE_EXIT),
+            &VmRequest::RegisterMemory(ref fd, size) => {
+                req.type_ = Le32::from(VM_REQUEST_TYPE_REGISTER_MEMORY);
+                req.size = Le64::from(size as u64);
+                fd_buf[0] = fd.as_raw_fd();
+                fd_len = 1;
+            }
+            &VmRequest::UnregisterMemory(slot) => {
+                req.type_ = Le32::from(VM_REQUEST_TYPE_UNREGISTER_MEMORY);
+                req.slot = Le32::from(slot);
+            }
+            _ => return Err(VmControlError::InvalidType),
+        }
+        let mut buf = [0; VM_REQUEST_SIZE];
+        buf.as_mut().get_ref(0).unwrap().store(req);
+        scm.send(s, &[buf.as_ref()], &fd_buf[..fd_len])
+            .map_err(|e| VmControlError::Send(e))?;
+        Ok(())
+    }
+
+    /// Executes this request on the given Vm and other mutable state.
+    ///
+    /// # Arguments
+    /// * `vm` - The `Vm` to perform the request on.
+    /// * `next_mem_pfn` - In/out argument for the page frame number to put the next chunk of device
+    /// memory into.
+    /// * `running` - Out argument that is set to false if the request was to stop running the VM.
+    ///
+    /// This does not return a result, instead encapsulating the success or failure in a
+    /// `VmResponse` with the intended purpose of sending the response back over the  socket that
+    /// received this `VmRequest`.
+    pub fn execute(&self, vm: &mut Vm, next_mem_pfn: &mut u64, running: &mut bool) -> VmResponse {
+        *running = true;
+        match self {
+            &VmRequest::Exit => {
+                *running = false;
+                VmResponse::Ok
+            }
+            &VmRequest::RegisterIoevent(ref evt, addr, datamatch) => {
+                match vm.register_ioevent(evt, addr, datamatch) {
+                    Ok(_) => VmResponse::Ok,
+                    Err(e) => VmResponse::Err(e),
+                }
+            }
+            &VmRequest::RegisterIrqfd(ref evt, irq) => {
+                match vm.register_irqfd(evt, irq) {
+                    Ok(_) => VmResponse::Ok,
+                    Err(e) => return VmResponse::Err(e),
+                }
+            }
+            &VmRequest::RegisterMemory(ref fd, size) => {
+                let mmap = match MemoryMapping::from_fd(fd, size) {
+                    Ok(v) => v,
+                    Err(MmapError::SystemCallFailed(e)) => return VmResponse::Err(e),
+                    _ => return VmResponse::Err(SysError::new(-EINVAL)),
+                };
+                let pfn = *next_mem_pfn;
+                let slot = match vm.add_device_memory(GuestAddress((pfn << 12) as usize), mmap) {
+                    Ok(slot) => slot,
+                    Err(e) => return VmResponse::Err(e),
+                };
+                // TODO(zachr): Use a smarter allocation strategy. The current strategy is just
+                // bumping this pointer, meaning the remove operation does not free any address
+                // space. Given enough allocations, device memory may run out of address space and
+                // collide with guest memory or MMIO address space. There is currently nothing in
+                // place to limit the amount of address space used by device memory.
+                *next_mem_pfn += (((size + 0x7ff) >> 12) + 1) as u64;
+                VmResponse::RegisterMemory {
+                    pfn: pfn,
+                    slot: slot,
+                }
+            }
+            &VmRequest::UnregisterMemory(slot) => {
+                match vm.remove_device_memory(slot) {
+                    Ok(_) => VmResponse::Ok,
+                    Err(e) => VmResponse::Err(e),
+                }
+            }
+        }
+    }
+}
+
+/// Indication of success or failure of a `VmRequest`.
+///
+/// Success is usually indicated `VmResponse::Ok` unless there is data associated with the response.
+#[derive(Debug, PartialEq)]
+pub enum VmResponse {
+    /// Indicates the request was executed successfully.
+    Ok,
+    /// Indicates the request encountered some error during execution.
+    Err(SysError),
+    /// The request to register memory into guest address space was successfully done at page frame
+    /// number `pfn` and memory slot number `slot`.
+    RegisterMemory { pfn: u64, slot: u32 },
+}
+
+const VM_RESPONSE_TYPE_OK: u32 = 1;
+const VM_RESPONSE_TYPE_ERR: u32 = 2;
+const VM_RESPONSE_TYPE_REGISTER_MEMORY: u32 = 3;
+const VM_RESPONSE_SIZE: usize = 24;
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct VmResponseStruct {
+    type_: Le32,
+    errno: Le32,
+    pfn: Le64,
+    slot: Le32,
+    padding: Le32,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl DataInit for VmResponseStruct {}
+
+impl VmResponse {
+    /// Receive a `VmResponse` from the given socket.
+    ///
+    /// This should be called after the sending a `VmRequest` before sending another request.
+    pub fn recv(scm: &mut Scm, s: &UnixDatagram) -> VmControlResult<VmResponse> {
+        let mut buf = [0; VM_RESPONSE_SIZE];
+        let mut fds = Vec::new();
+        let read = scm.recv(s, &mut [&mut buf], &mut fds)
+            .map_err(|e| VmControlError::Recv(e))?;
+        if read != VM_RESPONSE_SIZE {
+            return Err(VmControlError::BadSize(read));
+        }
+        let resp: VmResponseStruct = buf.as_mut().get_ref(0).unwrap().load();
+
+        match resp.type_.into() {
+            VM_RESPONSE_TYPE_OK => Ok(VmResponse::Ok),
+            VM_RESPONSE_TYPE_ERR => {
+                Ok(VmResponse::Err(SysError::new(-(resp.errno.to_native() as i32))))
+            }
+            VM_RESPONSE_TYPE_REGISTER_MEMORY => {
+                Ok(VmResponse::RegisterMemory {
+                       pfn: resp.pfn.into(),
+                       slot: resp.slot.into(),
+                   })
+            }
+            _ => Err(VmControlError::InvalidType),
+        }
+    }
+
+    /// Send a `VmResponse` over the given socket.
+    ///
+    /// This must be called after receiving a `VmRequest` to indicate the outcome of that request's
+    /// execution.
+    pub fn send(&self, scm: &mut Scm, s: &UnixDatagram) -> VmControlResult<()> {
+        let mut resp = VmResponseStruct::default();
+        match self {
+            &VmResponse::Ok => resp.type_ = Le32::from(VM_RESPONSE_TYPE_OK),
+            &VmResponse::Err(e) => {
+                resp.type_ = Le32::from(VM_RESPONSE_TYPE_ERR);
+                resp.errno = Le32::from(e.errno().checked_abs().unwrap_or(ERANGE) as u32);
+            }
+            &VmResponse::RegisterMemory { pfn, slot } => {
+                resp.type_ = Le32::from(VM_RESPONSE_TYPE_REGISTER_MEMORY);
+                resp.pfn = Le64::from(pfn);
+                resp.slot = Le32::from(slot);
+            }
+        }
+        let mut buf = [0; VM_RESPONSE_SIZE];
+        buf.as_mut().get_ref(0).unwrap().store(resp);
+        scm.send(s, &[buf.as_ref()], &[])
+            .map_err(|e| VmControlError::Send(e))?;
+        Ok(())
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::net::Shutdown;
+
+    use sys_util::SharedMemory;
+
+    #[test]
+    fn request_exit() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        VmRequest::Exit.send(&mut scm, &s1).unwrap();
+        match VmRequest::recv(&mut scm, &s2).unwrap() {
+            VmRequest::Exit => {}
+            _ => panic!("recv wrong request variant"),
+        }
+    }
+
+    #[test]
+    fn request_register_memory() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        let shm_size: usize = 4096;
+        let mut shm = SharedMemory::new(None).unwrap();
+        shm.set_size(shm_size as u64).unwrap();
+        VmRequest::RegisterMemory(MaybeOwnedFd::Borrowed(shm.as_raw_fd()), shm_size)
+            .send(&mut scm, &s1)
+            .unwrap();
+        match VmRequest::recv(&mut scm, &s2).unwrap() {
+            VmRequest::RegisterMemory(MaybeOwnedFd::Owned(fd), size) => {
+                assert!(fd.as_raw_fd() >= 0);
+                assert_eq!(size, shm_size);
+            }
+            _ => panic!("recv wrong request variant"),
+        }
+    }
+
+    #[test]
+    fn request_unregister_memory() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        VmRequest::UnregisterMemory(77)
+            .send(&mut scm, &s1)
+            .unwrap();
+        match VmRequest::recv(&mut scm, &s2).unwrap() {
+            VmRequest::UnregisterMemory(slot) => assert_eq!(slot, 77),
+            _ => panic!("recv wrong request variant"),
+        }
+    }
+
+    #[test]
+    fn request_expect_fd() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        let mut bad_request = [0; VM_REQUEST_SIZE];
+        bad_request[0] = VM_REQUEST_TYPE_REGISTER_MEMORY as u8;
+        scm.send(&s2, &[bad_request.as_ref()], &[]).unwrap();
+        match VmRequest::recv(&mut scm, &s1) {
+            Err(VmControlError::ExpectFd) => {}
+            _ => panic!("recv wrong error variant"),
+        }
+    }
+
+    #[test]
+    fn request_no_data() {
+        let (s1, _) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        s1.shutdown(Shutdown::Both).unwrap();
+        match VmRequest::recv(&mut scm, &s1) {
+            Err(VmControlError::BadSize(s)) => assert_eq!(s, 0),
+            _ => panic!("recv wrong error variant"),
+        }
+    }
+
+    #[test]
+    fn request_bad_size() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        scm.send(&s2, &[[12; 7].as_ref()], &[]).unwrap();
+        match VmRequest::recv(&mut scm, &s1) {
+            Err(VmControlError::BadSize(_)) => {}
+            _ => panic!("recv wrong error variant"),
+        }
+    }
+
+    #[test]
+    fn request_invalid_type() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        scm.send(&s2, &[[12; VM_RESPONSE_SIZE].as_ref()], &[])
+            .unwrap();
+        match VmRequest::recv(&mut scm, &s1) {
+            Err(VmControlError::InvalidType) => {}
+            _ => panic!("recv wrong error variant"),
+        }
+    }
+
+    #[test]
+    fn resp_ok() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        VmResponse::Ok.send(&mut scm, &s1).unwrap();
+        let r = VmResponse::recv(&mut scm, &s2).unwrap();
+        assert_eq!(r, VmResponse::Ok);
+    }
+
+    #[test]
+    fn resp_err() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        let r1 = VmResponse::Err(SysError::new(-89));
+        r1.send(&mut scm, &s1).unwrap();
+        let r2 = VmResponse::recv(&mut scm, &s2).unwrap();
+        assert_eq!(r1, r2);
+    }
+
+    #[test]
+    fn resp_memory() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        let r1 = VmResponse::RegisterMemory { pfn: 55, slot: 66 };
+        r1.send(&mut scm, &s1).unwrap();
+        let r2 = VmResponse::recv(&mut scm, &s2).unwrap();
+        assert_eq!(r1, r2);
+    }
+
+    #[test]
+    fn resp_no_data() {
+        let (s1, _) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        s1.shutdown(Shutdown::Both).unwrap();
+        let r = VmResponse::recv(&mut scm, &s1);
+        assert_eq!(r, Err(VmControlError::BadSize(0)));
+    }
+
+    #[test]
+    fn resp_bad_size() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        scm.send(&s2, &[[12; 7].as_ref()], &[]).unwrap();
+        let r = VmResponse::recv(&mut scm, &s1);
+        assert_eq!(r, Err(VmControlError::BadSize(7)));
+    }
+
+    #[test]
+    fn resp_invalid_type() {
+        let (s1, s2) = UnixDatagram::pair().expect("failed to create socket pair");
+        let mut scm = Scm::new(1);
+        scm.send(&s2, &[[12; VM_RESPONSE_SIZE].as_ref()], &[])
+            .unwrap();
+        let r = VmResponse::recv(&mut scm, &s1);
+        assert_eq!(r, Err(VmControlError::InvalidType));
+    }
+}
diff --git a/sys_util/src/lib.rs b/sys_util/src/lib.rs
index af9db2f..33f0b3e 100644
--- a/sys_util/src/lib.rs
+++ b/sys_util/src/lib.rs
@@ -46,5 +46,6 @@ pub use signalfd::*;
 pub use ioctl::*;
 pub use sock_ctrl_msg::*;
 
+pub use mmap::Error as MmapError;
 pub use guest_memory::Error as GuestMemoryError;
 pub use signalfd::Error as SignalFdError;