devices: implement MSI control socket

Allocate per device VmMsi msg_socket for communication between virtio
devices and main VM process, which owns the KVM fd and issues ioctl to
KVM for KVM_IRQFD and KVM_SET_GSI_ROUTING.

BUG=chromium:854765
TEST=None

Change-Id: Ie1c81534912eaab7fbf05b5edef7dca343db301c
Signed-off-by: Xiong Zhang <xiong.y.zhang@intel.corp-partner.google.com>
Signed-off-by: Zide Chen <zide.chen@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1828339
Tested-by: Daniel Verkamp <dverkamp@chromium.org>
Commit-Queue: Stephen Barber <smbarber@chromium.org>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>

3 files changed, 67 insertions, 13 deletions

diff --git a/devices/src/pci/msix.rs b/devices/src/pci/msix.rs
index eff033c..6110b72 100644
--- a/devices/src/pci/msix.rs
+++ b/devices/src/pci/msix.rs
@@ -3,8 +3,11 @@
 // found in the LICENSE file.
 
 use crate::pci::{PciCapability, PciCapabilityID};
+use msg_socket::{MsgReceiver, MsgSender};
 use std::convert::TryInto;
+use std::os::unix::io::{AsRawFd, RawFd};
 use sys_util::{error, EventFd};
+use vm_control::{MaybeOwnedFd, VmIrqRequest, VmIrqRequestSocket, VmIrqResponse};
 
 use data_model::DataInit;
 
@@ -52,11 +55,12 @@ pub struct MsixConfig {
     irq_vec: Vec<IrqfdGsi>,
     masked: bool,
     enabled: bool,
+    msi_device_socket: VmIrqRequestSocket,
     msix_num: u16,
 }
 
 impl MsixConfig {
-    pub fn new(msix_vectors: u16) -> Self {
+    pub fn new(msix_vectors: u16, vm_socket: VmIrqRequestSocket) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
 
         let mut table_entries: Vec<MsixTableEntry> = Vec::new();
@@ -71,6 +75,7 @@ impl MsixConfig {
             irq_vec: Vec::new(),
             masked: false,
             enabled: false,
+            msi_device_socket: vm_socket,
             msix_num: msix_vectors,
         }
     }
@@ -139,28 +144,46 @@ impl MsixConfig {
         }
     }
 
-    fn add_msi_route(&self, index: u16, _gsi: u32) {
+    fn add_msi_route(&self, index: u16, gsi: u32) {
         let mut data: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0];
         self.read_msix_table((index * 16).into(), data.as_mut());
         let msi_address: u64 = u64::from_le_bytes(data);
         let mut data: [u8; 4] = [0, 0, 0, 0];
         self.read_msix_table((index * 16 + 8).into(), data.as_mut());
-        let _msi_data: u32 = u32::from_le_bytes(data);
+        let msi_data: u32 = u32::from_le_bytes(data);
 
         if msi_address == 0 {
             return;
         }
 
-        // TODO: IPC to vm_control for VmIrqRequest::AddMsiRoute()
+        if let Err(e) = self.msi_device_socket.send(&VmIrqRequest::AddMsiRoute {
+            gsi,
+            msi_address,
+            msi_data,
+        }) {
+            error!("failed to send AddMsiRoute request: {:?}", e);
+            return;
+        }
+        if self.msi_device_socket.recv().is_err() {
+            error!("Faied to receive AddMsiRoute Response");
+        }
     }
 
     fn msix_enable(&mut self) {
         self.irq_vec.clear();
         for i in 0..self.msix_num {
             let irqfd = EventFd::new().unwrap();
-            let irq_num: u32 = 0;
-
-            // TODO: IPC to vm_control for VmIrqRequest::AllocateOneMsi()
+            if let Err(e) = self.msi_device_socket.send(&VmIrqRequest::AllocateOneMsi {
+                irqfd: MaybeOwnedFd::Borrowed(irqfd.as_raw_fd()),
+            }) {
+                error!("failed to send AllocateOneMsi request: {:?}", e);
+                continue;
+            }
+            let irq_num: u32;
+            match self.msi_device_socket.recv() {
+                Ok(VmIrqResponse::AllocateOneMsi { gsi }) => irq_num = gsi,
+                _ => continue,
+            }
             self.irq_vec.push(IrqfdGsi {
                 irqfd,
                 gsi: irq_num,
@@ -400,6 +423,11 @@ impl MsixConfig {
             irq.irqfd.write(1).unwrap();
         }
     }
+
+    /// Return the raw fd of the MSI device socket
+    pub fn get_msi_socket(&self) -> RawFd {
+        self.msi_device_socket.as_ref().as_raw_fd()
+    }
 }
 
 // It is safe to implement DataInit; all members are simple numbers and any value is valid.
diff --git a/devices/src/virtio/virtio_pci_device.rs b/devices/src/virtio/virtio_pci_device.rs
index 2110b75..713329f 100644
--- a/devices/src/virtio/virtio_pci_device.rs
+++ b/devices/src/virtio/virtio_pci_device.rs
@@ -19,6 +19,8 @@ use crate::pci::{
     PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciInterruptPin, PciSubclass,
 };
 
+use vm_control::VmIrqRequestSocket;
+
 use self::virtio_pci_common_config::VirtioPciCommonConfig;
 
 pub enum PciCapabilityType {
@@ -172,7 +174,11 @@ pub struct VirtioPciDevice {
 
 impl VirtioPciDevice {
     /// Constructs a new PCI transport for the given virtio device.
-    pub fn new(mem: GuestMemory, device: Box<dyn VirtioDevice>) -> Result<Self> {
+    pub fn new(
+        mem: GuestMemory,
+        device: Box<dyn VirtioDevice>,
+        msi_device_socket: Option<VmIrqRequestSocket>,
+    ) -> Result<Self> {
         let mut queue_evts = Vec::new();
         for _ in device.queue_max_sizes() {
             queue_evts.push(EventFd::new()?)
@@ -186,8 +192,11 @@ impl VirtioPciDevice {
         let pci_device_id = VIRTIO_PCI_DEVICE_ID_BASE + device.device_type() as u16;
 
         let msix_num = device.msix_vectors();
-        let msix_config = if msix_num > 0 {
-            let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_num)));
+        let msix_config = if msix_num > 0 && msi_device_socket.is_some() {
+            let msix_config = Arc::new(Mutex::new(MsixConfig::new(
+                msix_num,
+                msi_device_socket.unwrap(),
+            )));
             Some(msix_config)
         } else {
             None
@@ -346,6 +355,10 @@ impl PciDevice for VirtioPciDevice {
         if let Some(interrupt_resample_evt) = &self.interrupt_resample_evt {
             fds.push(interrupt_resample_evt.as_raw_fd());
         }
+        if let Some(msix_config) = &self.msix_config {
+            let fd = msix_config.lock().get_msi_socket();
+            fds.push(fd);
+        }
         fds
     }
 
diff --git a/src/linux.rs b/src/linux.rs
index 6de02e9..31b9d32 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -53,8 +53,9 @@ use vhost;
 use vm_control::{
     BalloonControlCommand, BalloonControlRequestSocket, BalloonControlResponseSocket,
     DiskControlCommand, DiskControlRequestSocket, DiskControlResponseSocket, DiskControlResult,
-    UsbControlSocket, VmControlResponseSocket, VmIrqResponseSocket, VmMemoryControlRequestSocket,
-    VmMemoryControlResponseSocket, VmMemoryRequest, VmMemoryResponse, VmRunMode,
+    UsbControlSocket, VmControlResponseSocket, VmIrqRequest, VmIrqResponse, VmIrqResponseSocket,
+    VmMemoryControlRequestSocket, VmMemoryControlResponseSocket, VmMemoryRequest, VmMemoryResponse,
+    VmRunMode,
 };
 
 use crate::{Config, DiskOption, Executable, TouchDeviceOption};
@@ -935,6 +936,7 @@ fn create_devices(
     vm: &mut Vm,
     resources: &mut SystemAllocator,
     exit_evt: &EventFd,
+    control_sockets: &mut Vec<TaggedControlSocket>,
     wayland_device_socket: VmMemoryControlRequestSocket,
     gpu_device_socket: VmMemoryControlRequestSocket,
     balloon_device_socket: BalloonControlResponseSocket,
@@ -956,7 +958,17 @@ fn create_devices(
     let mut pci_devices = Vec::new();
 
     for stub in stubs {
-        let dev = VirtioPciDevice::new(mem.clone(), stub.dev).map_err(Error::VirtioPciDev)?;
+        let dev = if stub.dev.msix_vectors() > 0 {
+            let (msi_host_socket, msi_device_socket) =
+                msg_socket::pair::<VmIrqResponse, VmIrqRequest>().map_err(Error::CreateSocket)?;
+            control_sockets.push(TaggedControlSocket::VmIrq(msi_host_socket));
+
+            VirtioPciDevice::new(mem.clone(), stub.dev, Some(msi_device_socket))
+                .map_err(Error::VirtioPciDev)?
+        } else {
+            VirtioPciDevice::new(mem.clone(), stub.dev, None).map_err(Error::VirtioPciDev)?
+        };
+
         let dev = Box::new(dev) as Box<dyn PciDevice>;
         pci_devices.push((dev, stub.jail));
     }
@@ -1363,6 +1375,7 @@ pub fn run_config(cfg: Config) -> Result<()> {
                 vm,
                 sys_allocator,
                 exit_evt,
+                &mut control_sockets,
                 wayland_device_socket,
                 gpu_device_socket,
                 balloon_device_socket,