hacky working out-of-process virtio_wl

12 files changed, 237 insertions, 26 deletions

diff --git a/Cargo.lock b/Cargo.lock
index d236770..2b6c00e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -184,6 +184,7 @@ dependencies = [
  "io_jail 0.1.0",
  "kvm 0.1.0",
  "kvm_sys 0.1.0",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.44 (registry+https://github.com/rust-lang/crates.io-index)",
  "libcras 0.1.0",
  "libvda 0.1.0",
@@ -404,6 +405,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "libc"
 version = "0.2.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -827,6 +833,7 @@ dependencies = [
 "checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9"
 "checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76"
 "checksum getopts 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0a7292d30132fb5424b354f5dc02512a86e4c516fe544bb7a25e7f266951b797"
+"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 "checksum libc 0.2.44 (registry+https://github.com/rust-lang/crates.io-index)" = "10923947f84a519a45c8fefb7dd1b3e8c08747993381adee176d7a82b4195311"
 "checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f"
 "checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
diff --git a/Cargo.toml b/Cargo.toml
index 55f7df0..0821dc0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,10 @@ path = "src/crosvm.rs"
 name = "crosvm"
 path = "src/main.rs"
 
+[[bin]]
+name = "crosvm_wl"
+path = "src/wl.rs"
+
 [profile.release]
 panic = 'abort'
 overflow-checks = true
diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs
index e80c934..d5d0066 100644
--- a/aarch64/src/lib.rs
+++ b/aarch64/src/lib.rs
@@ -183,12 +183,10 @@ impl std::error::Error for Error {}
 
 #[derive(Clone, Copy, Debug, MsgOnSocket)]
 pub struct MemoryParams {
-    size: u64,
+    pub size: u64,
 }
 
 impl MemoryParams {
-    // This should never be public to prevent architecture-specific code,
-    // but pub(crate) would be okay.
     fn new(components: &VmComponents) -> Self {
         MemoryParams {
             size: components.memory_size,
diff --git a/devices/Cargo.toml b/devices/Cargo.toml
index 8bea78d..a500cf8 100644
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -49,6 +49,7 @@ vfio_sys = { path = "../vfio_sys" }
 vhost = { path = "../vhost" }
 virtio_sys = { path = "../virtio_sys" }
 vm_control = { path = "../vm_control" }
+lazy_static = "1.4.0"
 
 [dev-dependencies]
 tempfile = { path = "../tempfile" }
diff --git a/devices/src/lib.rs b/devices/src/lib.rs
index 294a8cb..d648ac2 100644
--- a/devices/src/lib.rs
+++ b/devices/src/lib.rs
@@ -4,6 +4,9 @@
 
 //! Emulates virtual and hardware devices.
 
+#[macro_use]
+extern crate lazy_static;
+
 mod bus;
 mod cmos;
 mod i8042;
diff --git a/devices/src/virtio/queue.rs b/devices/src/virtio/queue.rs
index b76ee05..793246d 100644
--- a/devices/src/virtio/queue.rs
+++ b/devices/src/virtio/queue.rs
@@ -6,6 +6,7 @@ use std::cmp::min;
 use std::num::Wrapping;
 use std::sync::atomic::{fence, Ordering};
 
+use msg_socket::MsgOnSocket;
 use sys_util::{error, GuestAddress, GuestMemory};
 use virtio_sys::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 
@@ -199,7 +200,7 @@ impl<'a, 'b> Iterator for AvailIter<'a, 'b> {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug, MsgOnSocket)]
 /// A virtio queue's parameters.
 pub struct Queue {
     /// The maximal size in elements offered by the device
diff --git a/devices/src/virtio/virtio_pci_device.rs b/devices/src/virtio/virtio_pci_device.rs
index 92dd91e..0d44e48 100644
--- a/devices/src/virtio/virtio_pci_device.rs
+++ b/devices/src/virtio/virtio_pci_device.rs
@@ -404,6 +404,9 @@ impl PciDevice for VirtioPciDevice {
         if let Some(interrupt_resample_evt) = &self.interrupt_resample_evt {
             fds.push(interrupt_resample_evt.as_raw_fd());
         }
+        if let Some(mem) = &self.mem {
+            fds.push(mem.as_raw_fd());
+        }
         let fd = self.msix_config.lock().get_msi_socket();
         fds.push(fd);
         fds
diff --git a/devices/src/virtio/wl.rs b/devices/src/virtio/wl.rs
index 6fefd51..a685e51 100644
--- a/devices/src/virtio/wl.rs
+++ b/devices/src/virtio/wl.rs
@@ -67,10 +67,39 @@ use sys_util::ioctl_with_ref;
 
 use super::resource_bridge::*;
 use super::{
-    DescriptorChain, Interrupt, Queue, Reader, VirtioDevice, Writer, TYPE_WL, VIRTIO_F_VERSION_1,
+    DescriptorChain, Interrupt, InterruptProxyEvent, Queue, Reader, VirtioDevice, Writer, TYPE_WL,
+    VIRTIO_F_VERSION_1,
 };
 use vm_control::{MaybeOwnedFd, VmMemoryControlRequestSocket, VmMemoryRequest, VmMemoryResponse};
 
+use msg_socket::{MsgOnSocket, MsgSocket};
+use sys_util::net::UnixSeqpacket;
+
+#[derive(Debug, MsgOnSocket)]
+pub struct SingleFd {
+    pub memfd: MaybeOwnedFd,
+    pub interrupt: MaybeOwnedFd,
+    pub interrupt_resample_evt: MaybeOwnedFd,
+    pub in_queue: Queue,
+    pub out_queue: Queue,
+    pub vm_socket: MaybeOwnedFd,
+    pub use_transition_flags: bool,
+    pub in_queue_evt: MaybeOwnedFd,
+    pub out_queue_evt: MaybeOwnedFd,
+    pub kill_evt: MaybeOwnedFd,
+}
+
+type Socket = MsgSocket<SingleFd, ()>;
+
+lazy_static! {
+    static ref SOCKET: Socket = {
+        let mut path = std::env::var("XDG_RUNTIME_DIR").expect("XDG_RUNTIME_DIR missing");
+        path.push_str("/crosvm-wl.sock");
+        let socket = UnixSeqpacket::connect(&path).expect("connect failed");
+        MsgSocket::new(socket)
+    };
+}
+
 const VIRTWL_SEND_MAX_ALLOCS: usize = 28;
 const VIRTIO_WL_CMD_VFD_NEW: u32 = 256;
 const VIRTIO_WL_CMD_VFD_CLOSE: u32 = 257;
@@ -1336,7 +1365,7 @@ impl WlState {
     }
 }
 
-struct Worker {
+pub struct Worker {
     interrupt: Interrupt,
     mem: GuestMemory,
     in_queue: Queue,
@@ -1345,15 +1374,15 @@ struct Worker {
 }
 
 impl Worker {
-    fn new(
+    pub fn new(
         mem: GuestMemory,
         interrupt: Interrupt,
         in_queue: Queue,
         out_queue: Queue,
-        wayland_paths: Map<String, PathBuf>,
+        wayland_paths: Map<String, PathBuf>, // { "": "/run/user/1000/wayland-0" }
         vm_socket: VmMemoryControlRequestSocket,
         use_transition_flags: bool,
-        resource_bridge: Option<ResourceRequestSocket>,
+        resource_bridge: Option<ResourceRequestSocket>, // None
     ) -> Worker {
         Worker {
             interrupt,
@@ -1369,7 +1398,7 @@ impl Worker {
         }
     }
 
-    fn run(&mut self, mut queue_evts: Vec<EventFd>, kill_evt: EventFd) {
+    pub fn run(&mut self, mut queue_evts: Vec<EventFd>, kill_evt: EventFd) {
         let mut in_desc_chains: VecDeque<DescriptorChain> =
             VecDeque::with_capacity(QUEUE_SIZE as usize);
         let in_queue_evt = queue_evts.remove(0);
@@ -1529,6 +1558,34 @@ impl Worker {
     }
 }
 
+struct InterruptWorker {
+    socket: MsgSocket<(), InterruptProxyEvent>,
+    interrupt: Interrupt,
+}
+
+impl InterruptWorker {
+    fn new(socket: MsgSocket<(), InterruptProxyEvent>, interrupt: Interrupt) -> Self {
+        Self { socket, interrupt }
+    }
+
+    fn run(&self) {
+        loop {
+            use InterruptProxyEvent::*;
+            let val = self.socket.recv();
+            match val {
+                Ok(SignalUsedQueue(value)) => self.interrupt.signal_used_queue(value).unwrap(),
+                Ok(SignalConfigChanged) => self.interrupt.signal_config_changed().unwrap(),
+                Ok(InterruptResample) => self.interrupt.interrupt_resample().unwrap(),
+
+                Err(e) => {
+                    eprintln!("recv error: {}", e);
+                    panic!("recv error: {}", e)
+                }
+            }
+        }
+    }
+}
+
 pub struct Wl {
     kill_evt: Option<EventFd>,
     worker_thread: Option<thread::JoinHandle<()>>,
@@ -1579,6 +1636,12 @@ impl VirtioDevice for Wl {
             keep_fds.push(resource_bridge.as_raw_fd());
         }
 
+        if let Some(ref kill_evt) = self.kill_evt {
+            keep_fds.push(kill_evt.as_raw_fd());
+        }
+
+        keep_fds.push(SOCKET.as_raw_fd());
+
         keep_fds
     }
 
@@ -1624,21 +1687,32 @@ impl VirtioDevice for Wl {
             let wayland_paths = self.wayland_paths.clone();
             let use_transition_flags = self.use_transition_flags;
             let resource_bridge = self.resource_bridge.take();
+
+            let (ours, theirs) = UnixSeqpacket::pair().expect("pair failed");
+
+            if let Err(e) = SOCKET.send(&SingleFd {
+                memfd: MaybeOwnedFd::Borrowed(mem.as_raw_fd()),
+                interrupt: MaybeOwnedFd::Borrowed(theirs.as_raw_fd()),
+                interrupt_resample_evt: MaybeOwnedFd::Borrowed(
+                    interrupt.get_resample_evt().as_raw_fd(),
+                ),
+                in_queue: queues.remove(0),
+                out_queue: queues.remove(0),
+                vm_socket: MaybeOwnedFd::Borrowed(vm_socket.as_raw_fd()),
+                use_transition_flags,
+                in_queue_evt: MaybeOwnedFd::Borrowed(queue_evts[0].as_raw_fd()),
+                out_queue_evt: MaybeOwnedFd::Borrowed(queue_evts[1].as_raw_fd()),
+                kill_evt: MaybeOwnedFd::Borrowed(kill_evt.as_raw_fd()),
+            }) {
+                error!("failed to send SingleFd: {}", e);
+                return;
+            }
+
             let worker_result =
                 thread::Builder::new()
                     .name("virtio_wl".to_string())
                     .spawn(move || {
-                        Worker::new(
-                            mem,
-                            interrupt,
-                            queues.remove(0),
-                            queues.remove(0),
-                            wayland_paths,
-                            vm_socket,
-                            use_transition_flags,
-                            resource_bridge,
-                        )
-                        .run(queue_evts, kill_evt);
+                        InterruptWorker::new(MsgSocket::new(ours), interrupt).run();
                     });
 
             match worker_result {
diff --git a/seccomp/x86_64/wl_device.policy b/seccomp/x86_64/wl_device.policy
index f79b08a..e01c159 100644
--- a/seccomp/x86_64/wl_device.policy
+++ b/seccomp/x86_64/wl_device.policy
@@ -17,5 +17,8 @@ ftruncate: 1
 lseek: 1
 # Allow F_GETFL only
 fcntl: arg1 == 3
+# Used for socket to communicate with external wl.
+# arg0 == AF_UNIX && arg1 == SOCK_SEQPACKET|SOCK_CLOEXEC
+socketpair: arg0 == 1 && arg1 == 0x80005
 open: return ENOENT
 openat: return ENOENT
diff --git a/src/wl.rs b/src/wl.rs
new file mode 100644
index 0000000..419fb67
--- /dev/null
+++ b/src/wl.rs
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+use std::os::unix::prelude::*;
+
+use devices::virtio::{InterruptProxy, InterruptProxyEvent, SingleFd, Worker};
+use msg_socket::{MsgReceiver, MsgSocket};
+use std::collections::BTreeMap;
+use std::fs::remove_file;
+use sys_util::{
+    net::{UnixSeqpacket, UnixSeqpacketListener},
+    EventFd, GuestMemory, SharedMemory,
+};
+use vm_control::VmMemoryControlRequestSocket;
+
+#[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+pub use aarch64::{arch_memory_regions, MemoryParams};
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+pub use x86_64::{arch_memory_regions, MemoryParams};
+
+type Socket = MsgSocket<(), SingleFd>;
+
+fn main() {
+    eprintln!("hello world");
+
+    // Create and display the socket.
+    let mut path = std::env::var("XDG_RUNTIME_DIR").expect("XDG_RUNTIME_DIR missing");
+    path.push_str("/crosvm-wl.sock");
+    let _ = remove_file(&path);
+    let server = UnixSeqpacketListener::bind(&path).expect("failed to create control socket");
+    println!("{}", path);
+
+    // Receive connection from crosvm.
+    let conn = server.accept().expect("accept failed");
+    let msg_socket: Socket = MsgSocket::new(conn);
+
+    loop {
+        match msg_socket.recv() {
+            Ok(SingleFd {
+                memfd,
+                interrupt,
+                interrupt_resample_evt,
+                in_queue,
+                out_queue,
+                vm_socket,
+                use_transition_flags,
+                in_queue_evt,
+                out_queue_evt,
+                kill_evt,
+            }) => {
+                let shm = unsafe { SharedMemory::from_raw_fd(memfd.as_raw_fd()) };
+                std::mem::forget(memfd);
+
+                let regions = arch_memory_regions(MemoryParams {
+                    size: shm.size(),
+                    has_bios: false,
+                });
+                let mem =
+                    GuestMemory::with_memfd(&regions, shm).expect("GuestMemory::with_memfd failed");
+
+                let interrupt: MsgSocket<InterruptProxyEvent, ()> = MsgSocket::new(unsafe {
+                    let sock = UnixSeqpacket::from_raw_fd(interrupt.as_raw_fd());
+                    std::mem::forget(interrupt);
+                    sock
+                });
+                let vm_socket: VmMemoryControlRequestSocket = MsgSocket::new(unsafe {
+                    let sock = UnixSeqpacket::from_raw_fd(vm_socket.as_raw_fd());
+                    std::mem::forget(vm_socket);
+                    sock
+                });
+
+                let mut wayland_paths = BTreeMap::new();
+                wayland_paths.insert("".into(), "/run/user/1000/wayland-0".into());
+
+                let mut worker = Worker::new(
+                    mem,
+                    Box::new(InterruptProxy::new(interrupt, unsafe {
+                        let evt = EventFd::from_raw_fd(interrupt_resample_evt.as_raw_fd());
+                        std::mem::forget(interrupt_resample_evt);
+                        evt
+                    })),
+                    in_queue,
+                    out_queue,
+                    wayland_paths,
+                    vm_socket,
+                    use_transition_flags,
+                    None,
+                );
+
+                worker.run(
+                    vec![
+                        unsafe {
+                            let evt = EventFd::from_raw_fd(in_queue_evt.as_raw_fd());
+                            std::mem::forget(in_queue_evt);
+                            evt
+                        },
+                        unsafe {
+                            let evt = EventFd::from_raw_fd(out_queue_evt.as_raw_fd());
+                            std::mem::forget(out_queue_evt);
+                            evt
+                        },
+                    ],
+                    unsafe {
+                        let evt = EventFd::from_raw_fd(kill_evt.as_raw_fd());
+                        std::mem::forget(kill_evt);
+                        evt
+                    },
+                );
+            }
+
+            Err(e) => panic!("recv failed {:?}", e),
+        }
+    }
+}
diff --git a/sys_util/src/shm.rs b/sys_util/src/shm.rs
index ee5f5a3..04eca3e 100644
--- a/sys_util/src/shm.rs
+++ b/sys_util/src/shm.rs
@@ -211,6 +211,12 @@ impl SharedMemory {
     }
 }
 
+impl FromRawFd for SharedMemory {
+    unsafe fn from_raw_fd(fd: RawFd) -> Self {
+        Self::from_file(File::from_raw_fd(fd)).unwrap()
+    }
+}
+
 impl Read for SharedMemory {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         self.fd.read(buf)
diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs
index ba11080..09940cf 100644
--- a/x86_64/src/lib.rs
+++ b/x86_64/src/lib.rs
@@ -312,13 +312,11 @@ fn add_e820_entry(params: &mut boot_params, addr: u64, size: u64, mem_type: u32)
 #[derive(Clone, Copy, Debug, MsgOnSocket)]
 pub struct MemoryParams {
     /// Physical memory size in bytes for the VM.
-    size: u64,
-    has_bios: bool,
+    pub size: u64,
+    pub has_bios: bool,
 }
 
 impl MemoryParams {
-    // This should never be public to prevent architecture-specific code,
-    // but pub(crate) would be okay.
     fn new(components: &VmComponents) -> Self {
         let has_bios = match components.vm_image {
             VmImage::Bios(_) => true,
@@ -336,7 +334,7 @@ impl MemoryParams {
 /// These should be used to configure the GuestMemory structure for the platform.
 /// For x86_64 all addresses are valid from the start of the kernel except a
 /// carve out at the end of 32bit address space.
-fn arch_memory_regions(params: MemoryParams) -> Vec<(GuestAddress, u64)> {
+pub fn arch_memory_regions(params: MemoryParams) -> Vec<(GuestAddress, u64)> {
     let mem_end = GuestAddress(params.size);
     let first_addr_past_32bits = GuestAddress(FIRST_ADDR_PAST_32BITS);
     let end_32bit_gap_start = GuestAddress(END_ADDR_BEFORE_32BITS);