From 3d1bc2e0bb5bae7f32a9fa18b5348295facd5ab6 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Sat, 7 Mar 2020 19:29:32 +0000 Subject: hacky working out-of-process virtio_wl --- Cargo.lock | 7 ++ Cargo.toml | 4 ++ aarch64/src/lib.rs | 4 +- devices/Cargo.toml | 1 + devices/src/lib.rs | 3 + devices/src/virtio/queue.rs | 3 +- devices/src/virtio/virtio_pci_device.rs | 3 + devices/src/virtio/wl.rs | 108 +++++++++++++++++++++++++----- seccomp/x86_64/wl_device.policy | 3 + src/wl.rs | 113 ++++++++++++++++++++++++++++++++ sys_util/src/shm.rs | 6 ++ x86_64/src/lib.rs | 8 +-- 12 files changed, 237 insertions(+), 26 deletions(-) create mode 100644 src/wl.rs diff --git a/Cargo.lock b/Cargo.lock index d236770..2b6c00e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,7 @@ dependencies = [ "io_jail 0.1.0", "kvm 0.1.0", "kvm_sys 0.1.0", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.44 (registry+https://github.com/rust-lang/crates.io-index)", "libcras 0.1.0", "libvda 0.1.0", @@ -403,6 +404,11 @@ dependencies = [ "sys_util 0.1.0", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "libc" version = "0.2.44" @@ -827,6 +833,7 @@ dependencies = [ "checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" "checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" "checksum getopts 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0a7292d30132fb5424b354f5dc02512a86e4c516fe544bb7a25e7f266951b797" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum libc 0.2.44 (registry+https://github.com/rust-lang/crates.io-index)" = "10923947f84a519a45c8fefb7dd1b3e8c08747993381adee176d7a82b4195311" "checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f" "checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" diff --git a/Cargo.toml b/Cargo.toml index 55f7df0..0821dc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,10 @@ path = "src/crosvm.rs" name = "crosvm" path = "src/main.rs" +[[bin]] +name = "crosvm_wl" +path = "src/wl.rs" + [profile.release] panic = 'abort' overflow-checks = true diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs index e80c934..d5d0066 100644 --- a/aarch64/src/lib.rs +++ b/aarch64/src/lib.rs @@ -183,12 +183,10 @@ impl std::error::Error for Error {} #[derive(Clone, Copy, Debug, MsgOnSocket)] pub struct MemoryParams { - size: u64, + pub size: u64, } impl MemoryParams { - // This should never be public to prevent architecture-specific code, - // but pub(crate) would be okay. fn new(components: &VmComponents) -> Self { MemoryParams { size: components.memory_size, diff --git a/devices/Cargo.toml b/devices/Cargo.toml index 8bea78d..a500cf8 100644 --- a/devices/Cargo.toml +++ b/devices/Cargo.toml @@ -49,6 +49,7 @@ vfio_sys = { path = "../vfio_sys" } vhost = { path = "../vhost" } virtio_sys = { path = "../virtio_sys" } vm_control = { path = "../vm_control" } +lazy_static = "1.4.0" [dev-dependencies] tempfile = { path = "../tempfile" } diff --git a/devices/src/lib.rs b/devices/src/lib.rs index 294a8cb..d648ac2 100644 --- a/devices/src/lib.rs +++ b/devices/src/lib.rs @@ -4,6 +4,9 @@ //! Emulates virtual and hardware devices. +#[macro_use] +extern crate lazy_static; + mod bus; mod cmos; mod i8042; diff --git a/devices/src/virtio/queue.rs b/devices/src/virtio/queue.rs index b76ee05..793246d 100644 --- a/devices/src/virtio/queue.rs +++ b/devices/src/virtio/queue.rs @@ -6,6 +6,7 @@ use std::cmp::min; use std::num::Wrapping; use std::sync::atomic::{fence, Ordering}; +use msg_socket::MsgOnSocket; use sys_util::{error, GuestAddress, GuestMemory}; use virtio_sys::virtio_ring::VIRTIO_RING_F_EVENT_IDX; @@ -199,7 +200,7 @@ impl<'a, 'b> Iterator for AvailIter<'a, 'b> { } } -#[derive(Clone)] +#[derive(Clone, Debug, MsgOnSocket)] /// A virtio queue's parameters. pub struct Queue { /// The maximal size in elements offered by the device diff --git a/devices/src/virtio/virtio_pci_device.rs b/devices/src/virtio/virtio_pci_device.rs index 92dd91e..0d44e48 100644 --- a/devices/src/virtio/virtio_pci_device.rs +++ b/devices/src/virtio/virtio_pci_device.rs @@ -404,6 +404,9 @@ impl PciDevice for VirtioPciDevice { if let Some(interrupt_resample_evt) = &self.interrupt_resample_evt { fds.push(interrupt_resample_evt.as_raw_fd()); } + if let Some(mem) = &self.mem { + fds.push(mem.as_raw_fd()); + } let fd = self.msix_config.lock().get_msi_socket(); fds.push(fd); fds diff --git a/devices/src/virtio/wl.rs b/devices/src/virtio/wl.rs index 6fefd51..a685e51 100644 --- a/devices/src/virtio/wl.rs +++ b/devices/src/virtio/wl.rs @@ -67,10 +67,39 @@ use sys_util::ioctl_with_ref; use super::resource_bridge::*; use super::{ - DescriptorChain, Interrupt, Queue, Reader, VirtioDevice, Writer, TYPE_WL, VIRTIO_F_VERSION_1, + DescriptorChain, Interrupt, InterruptProxyEvent, Queue, Reader, VirtioDevice, Writer, TYPE_WL, + VIRTIO_F_VERSION_1, }; use vm_control::{MaybeOwnedFd, VmMemoryControlRequestSocket, VmMemoryRequest, VmMemoryResponse}; +use msg_socket::{MsgOnSocket, MsgSocket}; +use sys_util::net::UnixSeqpacket; + +#[derive(Debug, MsgOnSocket)] +pub struct SingleFd { + pub memfd: MaybeOwnedFd, + pub interrupt: MaybeOwnedFd, + pub interrupt_resample_evt: MaybeOwnedFd, + pub in_queue: Queue, + pub out_queue: Queue, + pub vm_socket: MaybeOwnedFd, + pub use_transition_flags: bool, + pub in_queue_evt: MaybeOwnedFd, + pub out_queue_evt: MaybeOwnedFd, + pub kill_evt: MaybeOwnedFd, +} + +type Socket = MsgSocket; + +lazy_static! { + static ref SOCKET: Socket = { + let mut path = std::env::var("XDG_RUNTIME_DIR").expect("XDG_RUNTIME_DIR missing"); + path.push_str("/crosvm-wl.sock"); + let socket = UnixSeqpacket::connect(&path).expect("connect failed"); + MsgSocket::new(socket) + }; +} + const VIRTWL_SEND_MAX_ALLOCS: usize = 28; const VIRTIO_WL_CMD_VFD_NEW: u32 = 256; const VIRTIO_WL_CMD_VFD_CLOSE: u32 = 257; @@ -1336,7 +1365,7 @@ impl WlState { } } -struct Worker { +pub struct Worker { interrupt: Interrupt, mem: GuestMemory, in_queue: Queue, @@ -1345,15 +1374,15 @@ struct Worker { } impl Worker { - fn new( + pub fn new( mem: GuestMemory, interrupt: Interrupt, in_queue: Queue, out_queue: Queue, - wayland_paths: Map, + wayland_paths: Map, // { "": "/run/user/1000/wayland-0" } vm_socket: VmMemoryControlRequestSocket, use_transition_flags: bool, - resource_bridge: Option, + resource_bridge: Option, // None ) -> Worker { Worker { interrupt, @@ -1369,7 +1398,7 @@ impl Worker { } } - fn run(&mut self, mut queue_evts: Vec, kill_evt: EventFd) { + pub fn run(&mut self, mut queue_evts: Vec, kill_evt: EventFd) { let mut in_desc_chains: VecDeque = VecDeque::with_capacity(QUEUE_SIZE as usize); let in_queue_evt = queue_evts.remove(0); @@ -1529,6 +1558,34 @@ impl Worker { } } +struct InterruptWorker { + socket: MsgSocket<(), InterruptProxyEvent>, + interrupt: Interrupt, +} + +impl InterruptWorker { + fn new(socket: MsgSocket<(), InterruptProxyEvent>, interrupt: Interrupt) -> Self { + Self { socket, interrupt } + } + + fn run(&self) { + loop { + use InterruptProxyEvent::*; + let val = self.socket.recv(); + match val { + Ok(SignalUsedQueue(value)) => self.interrupt.signal_used_queue(value).unwrap(), + Ok(SignalConfigChanged) => self.interrupt.signal_config_changed().unwrap(), + Ok(InterruptResample) => self.interrupt.interrupt_resample().unwrap(), + + Err(e) => { + eprintln!("recv error: {}", e); + panic!("recv error: {}", e) + } + } + } + } +} + pub struct Wl { kill_evt: Option, worker_thread: Option>, @@ -1579,6 +1636,12 @@ impl VirtioDevice for Wl { keep_fds.push(resource_bridge.as_raw_fd()); } + if let Some(ref kill_evt) = self.kill_evt { + keep_fds.push(kill_evt.as_raw_fd()); + } + + keep_fds.push(SOCKET.as_raw_fd()); + keep_fds } @@ -1624,21 +1687,32 @@ impl VirtioDevice for Wl { let wayland_paths = self.wayland_paths.clone(); let use_transition_flags = self.use_transition_flags; let resource_bridge = self.resource_bridge.take(); + + let (ours, theirs) = UnixSeqpacket::pair().expect("pair failed"); + + if let Err(e) = SOCKET.send(&SingleFd { + memfd: MaybeOwnedFd::Borrowed(mem.as_raw_fd()), + interrupt: MaybeOwnedFd::Borrowed(theirs.as_raw_fd()), + interrupt_resample_evt: MaybeOwnedFd::Borrowed( + interrupt.get_resample_evt().as_raw_fd(), + ), + in_queue: queues.remove(0), + out_queue: queues.remove(0), + vm_socket: MaybeOwnedFd::Borrowed(vm_socket.as_raw_fd()), + use_transition_flags, + in_queue_evt: MaybeOwnedFd::Borrowed(queue_evts[0].as_raw_fd()), + out_queue_evt: MaybeOwnedFd::Borrowed(queue_evts[1].as_raw_fd()), + kill_evt: MaybeOwnedFd::Borrowed(kill_evt.as_raw_fd()), + }) { + error!("failed to send SingleFd: {}", e); + return; + } + let worker_result = thread::Builder::new() .name("virtio_wl".to_string()) .spawn(move || { - Worker::new( - mem, - interrupt, - queues.remove(0), - queues.remove(0), - wayland_paths, - vm_socket, - use_transition_flags, - resource_bridge, - ) - .run(queue_evts, kill_evt); + InterruptWorker::new(MsgSocket::new(ours), interrupt).run(); }); match worker_result { diff --git a/seccomp/x86_64/wl_device.policy b/seccomp/x86_64/wl_device.policy index f79b08a..e01c159 100644 --- a/seccomp/x86_64/wl_device.policy +++ b/seccomp/x86_64/wl_device.policy @@ -17,5 +17,8 @@ ftruncate: 1 lseek: 1 # Allow F_GETFL only fcntl: arg1 == 3 +# Used for socket to communicate with external wl. +# arg0 == AF_UNIX && arg1 == SOCK_SEQPACKET|SOCK_CLOEXEC +socketpair: arg0 == 1 && arg1 == 0x80005 open: return ENOENT openat: return ENOENT diff --git a/src/wl.rs b/src/wl.rs new file mode 100644 index 0000000..419fb67 --- /dev/null +++ b/src/wl.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: BSD-3-Clause + +use std::os::unix::prelude::*; + +use devices::virtio::{InterruptProxy, InterruptProxyEvent, SingleFd, Worker}; +use msg_socket::{MsgReceiver, MsgSocket}; +use std::collections::BTreeMap; +use std::fs::remove_file; +use sys_util::{ + net::{UnixSeqpacket, UnixSeqpacketListener}, + EventFd, GuestMemory, SharedMemory, +}; +use vm_control::VmMemoryControlRequestSocket; + +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +pub use aarch64::{arch_memory_regions, MemoryParams}; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub use x86_64::{arch_memory_regions, MemoryParams}; + +type Socket = MsgSocket<(), SingleFd>; + +fn main() { + eprintln!("hello world"); + + // Create and display the socket. + let mut path = std::env::var("XDG_RUNTIME_DIR").expect("XDG_RUNTIME_DIR missing"); + path.push_str("/crosvm-wl.sock"); + let _ = remove_file(&path); + let server = UnixSeqpacketListener::bind(&path).expect("failed to create control socket"); + println!("{}", path); + + // Receive connection from crosvm. + let conn = server.accept().expect("accept failed"); + let msg_socket: Socket = MsgSocket::new(conn); + + loop { + match msg_socket.recv() { + Ok(SingleFd { + memfd, + interrupt, + interrupt_resample_evt, + in_queue, + out_queue, + vm_socket, + use_transition_flags, + in_queue_evt, + out_queue_evt, + kill_evt, + }) => { + let shm = unsafe { SharedMemory::from_raw_fd(memfd.as_raw_fd()) }; + std::mem::forget(memfd); + + let regions = arch_memory_regions(MemoryParams { + size: shm.size(), + has_bios: false, + }); + let mem = + GuestMemory::with_memfd(®ions, shm).expect("GuestMemory::with_memfd failed"); + + let interrupt: MsgSocket = MsgSocket::new(unsafe { + let sock = UnixSeqpacket::from_raw_fd(interrupt.as_raw_fd()); + std::mem::forget(interrupt); + sock + }); + let vm_socket: VmMemoryControlRequestSocket = MsgSocket::new(unsafe { + let sock = UnixSeqpacket::from_raw_fd(vm_socket.as_raw_fd()); + std::mem::forget(vm_socket); + sock + }); + + let mut wayland_paths = BTreeMap::new(); + wayland_paths.insert("".into(), "/run/user/1000/wayland-0".into()); + + let mut worker = Worker::new( + mem, + Box::new(InterruptProxy::new(interrupt, unsafe { + let evt = EventFd::from_raw_fd(interrupt_resample_evt.as_raw_fd()); + std::mem::forget(interrupt_resample_evt); + evt + })), + in_queue, + out_queue, + wayland_paths, + vm_socket, + use_transition_flags, + None, + ); + + worker.run( + vec![ + unsafe { + let evt = EventFd::from_raw_fd(in_queue_evt.as_raw_fd()); + std::mem::forget(in_queue_evt); + evt + }, + unsafe { + let evt = EventFd::from_raw_fd(out_queue_evt.as_raw_fd()); + std::mem::forget(out_queue_evt); + evt + }, + ], + unsafe { + let evt = EventFd::from_raw_fd(kill_evt.as_raw_fd()); + std::mem::forget(kill_evt); + evt + }, + ); + } + + Err(e) => panic!("recv failed {:?}", e), + } + } +} diff --git a/sys_util/src/shm.rs b/sys_util/src/shm.rs index ee5f5a3..04eca3e 100644 --- a/sys_util/src/shm.rs +++ b/sys_util/src/shm.rs @@ -211,6 +211,12 @@ impl SharedMemory { } } +impl FromRawFd for SharedMemory { + unsafe fn from_raw_fd(fd: RawFd) -> Self { + Self::from_file(File::from_raw_fd(fd)).unwrap() + } +} + impl Read for SharedMemory { fn read(&mut self, buf: &mut [u8]) -> io::Result { self.fd.read(buf) diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs index ba11080..09940cf 100644 --- a/x86_64/src/lib.rs +++ b/x86_64/src/lib.rs @@ -312,13 +312,11 @@ fn add_e820_entry(params: &mut boot_params, addr: u64, size: u64, mem_type: u32) #[derive(Clone, Copy, Debug, MsgOnSocket)] pub struct MemoryParams { /// Physical memory size in bytes for the VM. - size: u64, - has_bios: bool, + pub size: u64, + pub has_bios: bool, } impl MemoryParams { - // This should never be public to prevent architecture-specific code, - // but pub(crate) would be okay. fn new(components: &VmComponents) -> Self { let has_bios = match components.vm_image { VmImage::Bios(_) => true, @@ -336,7 +334,7 @@ impl MemoryParams { /// These should be used to configure the GuestMemory structure for the platform. /// For x86_64 all addresses are valid from the start of the kernel except a /// carve out at the end of 32bit address space. -fn arch_memory_regions(params: MemoryParams) -> Vec<(GuestAddress, u64)> { +pub fn arch_memory_regions(params: MemoryParams) -> Vec<(GuestAddress, u64)> { let mem_end = GuestAddress(params.size); let first_addr_past_32bits = GuestAddress(FIRST_ADDR_PAST_32BITS); let end_32bit_gap_start = GuestAddress(END_ADDR_BEFORE_32BITS); -- cgit 1.4.1