devices: implement MsixConfig struct and generic MSI-X functions

The MsixConfig struct is responsible for all the operations of MSI-X
Capability Structure and MSI-X Table.

A msix_config object is created for each virtio device.

BUG=chromium:854765
TEST=cargo test -p devices

Change-Id: Ide7c34d335d49a201f20b0a4307bcda97d1d61b7
Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
Signed-off-by: Xiong Zhang <xiong.y.zhang@intel.corp-partner.google.com>
Signed-off-by: Zide Chen <zide.chen@intel.corp-partner.google.com>
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1828337
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Tested-by: Daniel Verkamp <dverkamp@chromium.org>
Commit-Queue: Stephen Barber <smbarber@chromium.org>

17 files changed, 401 insertions, 8 deletions

diff --git a/devices/src/pci/mod.rs b/devices/src/pci/mod.rs
index eee87a5..a9e8749 100644
--- a/devices/src/pci/mod.rs
+++ b/devices/src/pci/mod.rs
@@ -15,7 +15,7 @@ mod pci_root;
 mod vfio_pci;
 
 pub use self::ac97::Ac97Dev;
-pub use self::msix::MsixCap;
+pub use self::msix::{MsixCap, MsixConfig};
 pub use self::pci_configuration::{
     PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityID,
     PciClassCode, PciConfiguration, PciHeaderType, PciProgrammingInterface, PciSerialBusSubClass,
diff --git a/devices/src/pci/msix.rs b/devices/src/pci/msix.rs
index 55e71ba..f1b7875 100644
--- a/devices/src/pci/msix.rs
+++ b/devices/src/pci/msix.rs
@@ -3,10 +3,288 @@
 // found in the LICENSE file.
 
 use crate::pci::{PciCapability, PciCapabilityID};
+use std::convert::TryInto;
+use sys_util::error;
 
 use data_model::DataInit;
 
 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
+const MSIX_TABLE_ENTRIES_MODULO: u64 = 16;
+const MSIX_PBA_ENTRIES_MODULO: u64 = 8;
+const BITS_PER_PBA_ENTRY: usize = 64;
+const FUNCTION_MASK_BIT: u16 = 0x4000;
+const MSIX_ENABLE_BIT: u16 = 0x8000;
+
+#[derive(Clone)]
+struct MsixTableEntry {
+    msg_addr_lo: u32,
+    msg_addr_hi: u32,
+    msg_data: u32,
+    vector_ctl: u32,
+}
+
+impl MsixTableEntry {
+    #[allow(dead_code)]
+    fn masked(&self) -> bool {
+        self.vector_ctl & 0x1 == 0x1
+    }
+}
+
+impl Default for MsixTableEntry {
+    fn default() -> Self {
+        MsixTableEntry {
+            msg_addr_lo: 0,
+            msg_addr_hi: 0,
+            msg_data: 0,
+            vector_ctl: 0,
+        }
+    }
+}
+
+/// Wrapper over MSI-X Capability Structure and MSI-X Tables
+pub struct MsixConfig {
+    table_entries: Vec<MsixTableEntry>,
+    pba_entries: Vec<u64>,
+    masked: bool,
+    enabled: bool,
+    _msix_num: u16,
+}
+
+impl MsixConfig {
+    pub fn new(msix_vectors: u16) -> Self {
+        assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
+
+        let mut table_entries: Vec<MsixTableEntry> = Vec::new();
+        table_entries.resize_with(msix_vectors as usize, Default::default);
+        let mut pba_entries: Vec<u64> = Vec::new();
+        let num_pba_entries: usize = ((msix_vectors as usize) / BITS_PER_PBA_ENTRY) + 1;
+        pba_entries.resize_with(num_pba_entries, Default::default);
+
+        MsixConfig {
+            table_entries,
+            pba_entries,
+            masked: false,
+            enabled: false,
+            _msix_num: msix_vectors,
+        }
+    }
+
+    /// Check whether the Function Mask bit in Message Control word in set or not.
+    /// if 1, all of the vectors associated with the function are masked,
+    /// regardless of their per-vector Mask bit states.
+    /// If 0, each vector’s Mask bit determines whether the vector is masked or not.
+    pub fn masked(&self) -> bool {
+        self.masked
+    }
+
+    /// Check whether the MSI-X Enable bit in Message Control word in set or not.
+    /// if 1, the function is permitted to use MSI-X to request service.
+    pub fn enabled(&self) -> bool {
+        self.enabled
+    }
+
+    /// Read the MSI-X Capability Structure.
+    /// The top 2 bits in Message Control word are emulated and all other
+    /// bits are read only.
+    pub fn read_msix_capability(&self, data: u32) -> u32 {
+        let mut msg_ctl = (data >> 16) as u16;
+        msg_ctl &= !(MSIX_ENABLE_BIT | FUNCTION_MASK_BIT);
+
+        if self.enabled {
+            msg_ctl |= MSIX_ENABLE_BIT;
+        }
+        if self.masked {
+            msg_ctl |= FUNCTION_MASK_BIT;
+        }
+        (msg_ctl as u32) << 16 | (data & u16::max_value() as u32)
+    }
+
+    /// Write to the MSI-X Capability Structure.
+    /// Only the top 2 bits in Message Control Word are writable.
+    pub fn write_msix_capability(&mut self, offset: u64, data: &[u8]) {
+        if offset == 2 && data.len() == 2 {
+            let reg = u16::from_le_bytes([data[0], data[1]]);
+
+            self.masked = (reg & FUNCTION_MASK_BIT) == FUNCTION_MASK_BIT;
+            self.enabled = (reg & MSIX_ENABLE_BIT) == MSIX_ENABLE_BIT;
+        } else {
+            error!(
+                "invalid write to MSI-X Capability Structure offset {:x}",
+                offset
+            );
+        }
+    }
+
+    /// Read MSI-X table
+    ///  # Arguments
+    ///  * 'offset' - the offset within the MSI-X Table
+    ///  * 'data' - used to store the read results
+    ///
+    /// For all accesses to MSI-X Table and MSI-X PBA fields, software must use aligned full
+    /// DWORD or aligned full QWORD transactions; otherwise, the result is undefined.
+    ///
+    ///             DWORD3            DWORD2      DWORD1            DWORD0
+    ///   entry 0:  Vector Control    Msg Data    Msg Upper Addr    Msg Addr
+    ///   entry 1:  Vector Control    Msg Data    Msg Upper Addr    Msg Addr
+    ///   entry 2:  Vector Control    Msg Data    Msg Upper Addr    Msg Addr
+    ///   ...
+    pub fn read_msix_table(&self, offset: u64, data: &mut [u8]) {
+        let index: usize = (offset / MSIX_TABLE_ENTRIES_MODULO) as usize;
+        let modulo_offset = offset % MSIX_TABLE_ENTRIES_MODULO;
+
+        match data.len() {
+            4 => {
+                let value = match modulo_offset {
+                    0x0 => self.table_entries[index].msg_addr_lo,
+                    0x4 => self.table_entries[index].msg_addr_hi,
+                    0x8 => self.table_entries[index].msg_data,
+                    0xc => self.table_entries[index].vector_ctl,
+                    _ => {
+                        error!("invalid offset");
+                        0
+                    }
+                };
+
+                data.copy_from_slice(&value.to_le_bytes());
+            }
+            8 => {
+                let value = match modulo_offset {
+                    0x0 => {
+                        (u64::from(self.table_entries[index].msg_addr_hi) << 32)
+                            | u64::from(self.table_entries[index].msg_addr_lo)
+                    }
+                    0x8 => {
+                        (u64::from(self.table_entries[index].vector_ctl) << 32)
+                            | u64::from(self.table_entries[index].msg_data)
+                    }
+                    _ => {
+                        error!("invalid offset");
+                        0
+                    }
+                };
+
+                data.copy_from_slice(&value.to_le_bytes());
+            }
+            _ => error!("invalid data length"),
+        };
+    }
+
+    /// Write to MSI-X table
+    ///
+    /// Message Address: the contents of this field specifies the address
+    ///     for the memory write transaction; different MSI-X vectors have
+    ///     different Message Address values
+    /// Message Data: the contents of this field specifies the data driven
+    ///     on AD[31::00] during the memory write transaction’s data phase.
+    /// Vector Control: only bit 0 (Mask Bit) is not reserved: when this bit
+    ///     is set, the function is prohibited from sending a message using
+    ///     this MSI-X Table entry.
+    pub fn write_msix_table(&mut self, offset: u64, data: &[u8]) {
+        let index: usize = (offset / MSIX_TABLE_ENTRIES_MODULO) as usize;
+        let modulo_offset = offset % MSIX_TABLE_ENTRIES_MODULO;
+
+        match data.len() {
+            4 => {
+                let value = u32::from_le_bytes(data.try_into().unwrap());
+                match modulo_offset {
+                    0x0 => self.table_entries[index].msg_addr_lo = value,
+                    0x4 => self.table_entries[index].msg_addr_hi = value,
+                    0x8 => self.table_entries[index].msg_data = value,
+                    0xc => self.table_entries[index].vector_ctl = value,
+                    _ => error!("invalid offset"),
+                };
+            }
+            8 => {
+                let value = u64::from_le_bytes(data.try_into().unwrap());
+                match modulo_offset {
+                    0x0 => {
+                        self.table_entries[index].msg_addr_lo = (value & 0xffff_ffffu64) as u32;
+                        self.table_entries[index].msg_addr_hi = (value >> 32) as u32;
+                    }
+                    0x8 => {
+                        self.table_entries[index].msg_data = (value & 0xffff_ffffu64) as u32;
+                        self.table_entries[index].vector_ctl = (value >> 32) as u32;
+                    }
+                    _ => error!("invalid offset"),
+                };
+            }
+            _ => error!("invalid data length"),
+        };
+    }
+
+    /// Read PBA Entries
+    ///  # Arguments
+    ///  * 'offset' - the offset within the PBA entries
+    ///  * 'data' - used to store the read results
+    ///
+    /// Pending Bits[63::00]: For each Pending Bit that is set, the function
+    /// has a pending message for the associated MSI-X Table entry.
+    pub fn read_pba_entries(&self, offset: u64, data: &mut [u8]) {
+        let index: usize = (offset / MSIX_PBA_ENTRIES_MODULO) as usize;
+        let modulo_offset = offset % MSIX_PBA_ENTRIES_MODULO;
+
+        match data.len() {
+            4 => {
+                let value: u32 = match modulo_offset {
+                    0x0 => (self.pba_entries[index] & 0xffff_ffffu64) as u32,
+                    0x4 => (self.pba_entries[index] >> 32) as u32,
+                    _ => {
+                        error!("invalid offset");
+                        0
+                    }
+                };
+
+                data.copy_from_slice(&value.to_le_bytes());
+            }
+            8 => {
+                let value: u64 = match modulo_offset {
+                    0x0 => self.pba_entries[index],
+                    _ => {
+                        error!("invalid offset");
+                        0
+                    }
+                };
+
+                data.copy_from_slice(&value.to_le_bytes());
+            }
+            _ => error!("invalid data length"),
+        }
+    }
+
+    /// Write to PBA Entries
+    ///
+    /// Software should never write, and should only read Pending Bits.
+    /// If software writes to Pending Bits, the result is undefined.
+    pub fn write_pba_entries(&mut self, _offset: u64, _data: &[u8]) {
+        error!("Pending Bit Array is read only");
+    }
+
+    #[allow(dead_code)]
+    fn set_pba_bit(&mut self, vector: u16, set: bool) {
+        assert!(vector < MAX_MSIX_VECTORS_PER_DEVICE);
+
+        let index: usize = (vector as usize) / BITS_PER_PBA_ENTRY;
+        let shift: usize = (vector as usize) % BITS_PER_PBA_ENTRY;
+        let mut mask: u64 = (1 << shift) as u64;
+
+        if set {
+            self.pba_entries[index] |= mask;
+        } else {
+            mask = !mask;
+            self.pba_entries[index] &= mask;
+        }
+    }
+
+    #[allow(dead_code)]
+    fn get_pba_bit(&self, vector: u16) -> u8 {
+        assert!(vector < MAX_MSIX_VECTORS_PER_DEVICE);
+
+        let index: usize = (vector as usize) / BITS_PER_PBA_ENTRY;
+        let shift: usize = (vector as usize) % BITS_PER_PBA_ENTRY;
+
+        ((self.pba_entries[index] >> shift) & 0x0000_0001u64) as u8
+    }
+}
 
 // It is safe to implement DataInit; all members are simple numbers and any value is valid.
 unsafe impl DataInit for MsixCap {}
@@ -56,7 +334,7 @@ impl MsixCap {
         assert!(table_size < MAX_MSIX_VECTORS_PER_DEVICE);
 
         // Set the table size and enable MSI-X.
-        let msg_ctl: u16 = 0x8000u16 + table_size - 1;
+        let msg_ctl: u16 = MSIX_ENABLE_BIT + table_size - 1;
 
         MsixCap {
             _cap_vndr: 0,
diff --git a/devices/src/virtio/balloon.rs b/devices/src/virtio/balloon.rs
index ec16f88..4f2e692 100644
--- a/devices/src/virtio/balloon.rs
+++ b/devices/src/virtio/balloon.rs
@@ -8,7 +8,9 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
+use crate::pci::MsixConfig;
 use data_model::{DataInit, Le32};
 use msg_socket::MsgReceiver;
 use sys_util::{
@@ -340,6 +342,7 @@ impl VirtioDevice for Balloon {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/block.rs b/devices/src/virtio/block.rs
index a76a3b3..554898c 100644
--- a/devices/src/virtio/block.rs
+++ b/devices/src/virtio/block.rs
@@ -13,6 +13,7 @@ use std::thread;
 use std::time::Duration;
 use std::u32;
 
+use crate::pci::MsixConfig;
 use data_model::{DataInit, Le16, Le32, Le64};
 use disk::DiskFile;
 use msg_socket::{MsgReceiver, MsgSender};
@@ -785,6 +786,7 @@ impl VirtioDevice for Block {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/gpu/mod.rs b/devices/src/virtio/gpu/mod.rs
index 97c84d5..8be0e3b 100644
--- a/devices/src/virtio/gpu/mod.rs
+++ b/devices/src/virtio/gpu/mod.rs
@@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
+use sync::Mutex;
 
 use data_model::*;
 
@@ -37,7 +38,7 @@ use super::{
 
 use self::backend::Backend;
 use self::protocol::*;
-use crate::pci::{PciBarConfiguration, PciBarPrefetchable, PciBarRegionType};
+use crate::pci::{MsixConfig, PciBarConfiguration, PciBarPrefetchable, PciBarRegionType};
 
 use vm_control::VmMemoryControlRequestSocket;
 
@@ -864,6 +865,7 @@ impl VirtioDevice for Gpu {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         interrupt_status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/input/mod.rs b/devices/src/virtio/input/mod.rs
index 2459d24..2694d46 100644
--- a/devices/src/virtio/input/mod.rs
+++ b/devices/src/virtio/input/mod.rs
@@ -28,6 +28,9 @@ use std::mem::size_of;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
+
+use crate::pci::MsixConfig;
 
 const EVENT_QUEUE_SIZE: u16 = 64;
 const STATUS_QUEUE_SIZE: u16 = 64;
@@ -621,6 +624,7 @@ where
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/net.rs b/devices/src/virtio/net.rs
index 3fb3ec1..4cb2741 100644
--- a/devices/src/virtio/net.rs
+++ b/devices/src/virtio/net.rs
@@ -10,7 +10,9 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
+use crate::pci::MsixConfig;
 use libc::{EAGAIN, EEXIST};
 use net_sys;
 use net_util::{Error as TapError, MacAddress, TapT};
@@ -491,6 +493,7 @@ where
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/p9.rs b/devices/src/virtio/p9.rs
index 6d89a45..83aa84e 100644
--- a/devices/src/virtio/p9.rs
+++ b/devices/src/virtio/p9.rs
@@ -11,7 +11,9 @@ use std::result;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
+use crate::pci::MsixConfig;
 use p9;
 use sys_util::{error, warn, Error as SysError, EventFd, GuestMemory, PollContext, PollToken};
 use virtio_sys::vhost::VIRTIO_F_VERSION_1;
@@ -239,6 +241,7 @@ impl VirtioDevice for P9 {
         guest_mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/pmem.rs b/devices/src/virtio/pmem.rs
index 4df295b..bc96f94 100644
--- a/devices/src/virtio/pmem.rs
+++ b/devices/src/virtio/pmem.rs
@@ -9,12 +9,15 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
 use sys_util::Result as SysResult;
 use sys_util::{error, EventFd, GuestAddress, GuestMemory, PollContext, PollToken};
 
 use data_model::{DataInit, Le32, Le64};
 
+use crate::pci::MsixConfig;
+
 use super::{
     copy_config, DescriptorChain, DescriptorError, Queue, Reader, VirtioDevice, Writer,
     INTERRUPT_STATUS_USED_RING, TYPE_PMEM, VIRTIO_F_VERSION_1,
@@ -278,6 +281,7 @@ impl VirtioDevice for Pmem {
         memory: GuestMemory,
         interrupt_event: EventFd,
         interrupt_resample_event: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_events: Vec<EventFd>,
diff --git a/devices/src/virtio/rng.rs b/devices/src/virtio/rng.rs
index 17dca69..41e1b4b 100644
--- a/devices/src/virtio/rng.rs
+++ b/devices/src/virtio/rng.rs
@@ -10,11 +10,14 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
 use sys_util::{error, warn, EventFd, GuestMemory, PollContext, PollToken};
 
 use super::{Queue, VirtioDevice, Writer, INTERRUPT_STATUS_USED_RING, TYPE_RNG};
 
+use crate::pci::MsixConfig;
+
 const QUEUE_SIZE: u16 = 256;
 const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
 
@@ -187,6 +190,7 @@ impl VirtioDevice for Rng {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/tpm.rs b/devices/src/virtio/tpm.rs
index 4e557dc..2267fed 100644
--- a/devices/src/virtio/tpm.rs
+++ b/devices/src/virtio/tpm.rs
@@ -13,6 +13,8 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
 
+use crate::pci::MsixConfig;
+use sync::Mutex;
 use sys_util::{error, EventFd, GuestMemory, PollContext, PollToken};
 use tpm2;
 
@@ -216,6 +218,7 @@ impl VirtioDevice for Tpm {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         interrupt_status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/vhost/net.rs b/devices/src/virtio/vhost/net.rs
index dfa9030..229b4f6 100644
--- a/devices/src/virtio/vhost/net.rs
+++ b/devices/src/virtio/vhost/net.rs
@@ -8,6 +8,7 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
 use net_sys;
 use net_util::{MacAddress, TapT};
@@ -18,6 +19,7 @@ use virtio_sys::virtio_net;
 
 use super::worker::Worker;
 use super::{Error, Result};
+use crate::pci::MsixConfig;
 use crate::virtio::{Queue, VirtioDevice, TYPE_NET};
 
 const QUEUE_SIZE: u16 = 256;
@@ -171,6 +173,7 @@ where
         _: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
@@ -288,6 +291,7 @@ pub mod tests {
             guest_memory,
             EventFd::new().unwrap(),
             EventFd::new().unwrap(),
+            None,
             Arc::new(AtomicUsize::new(0)),
             vec![Queue::new(1)],
             vec![EventFd::new().unwrap()],
diff --git a/devices/src/virtio/vhost/vsock.rs b/devices/src/virtio/vhost/vsock.rs
index d9bcda7..b39c368 100644
--- a/devices/src/virtio/vhost/vsock.rs
+++ b/devices/src/virtio/vhost/vsock.rs
@@ -6,6 +6,7 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;
 use std::thread;
+use sync::Mutex;
 
 use data_model::{DataInit, Le64};
 
@@ -14,6 +15,7 @@ use vhost::Vsock as VhostVsockHandle;
 
 use super::worker::Worker;
 use super::{Error, Result};
+use crate::pci::MsixConfig;
 use crate::virtio::{copy_config, Queue, VirtioDevice, TYPE_VSOCK};
 
 const QUEUE_SIZE: u16 = 256;
@@ -138,6 +140,7 @@ impl VirtioDevice for Vsock {
         _: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/virtio_device.rs b/devices/src/virtio/virtio_device.rs
index fa80b48..de4e24c 100644
--- a/devices/src/virtio/virtio_device.rs
+++ b/devices/src/virtio/virtio_device.rs
@@ -5,11 +5,12 @@
 use std::os::unix::io::RawFd;
 use std::sync::atomic::AtomicUsize;
 use std::sync::Arc;
+use sync::Mutex;
 
 use sys_util::{EventFd, GuestMemory};
 
 use super::*;
-use crate::pci::{PciBarConfiguration, PciCapability};
+use crate::pci::{MsixConfig, PciBarConfiguration, PciCapability};
 
 /// Trait for virtio devices to be driven by a virtio transport.
 ///
@@ -70,6 +71,7 @@ pub trait VirtioDevice: Send {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
diff --git a/devices/src/virtio/virtio_pci_common_config.rs b/devices/src/virtio/virtio_pci_common_config.rs
index ca9b662..977e2c0 100644
--- a/devices/src/virtio/virtio_pci_common_config.rs
+++ b/devices/src/virtio/virtio_pci_common_config.rs
@@ -36,6 +36,7 @@ pub struct VirtioPciCommonConfig {
     pub device_feature_select: u32,
     pub driver_feature_select: u32,
     pub queue_select: u16,
+    pub msix_config: u16,
 }
 
 impl VirtioPciCommonConfig {
@@ -236,9 +237,11 @@ impl VirtioPciCommonConfig {
 mod tests {
     use super::*;
 
+    use crate::pci::MsixConfig;
     use std::os::unix::io::RawFd;
     use std::sync::atomic::AtomicUsize;
     use std::sync::Arc;
+    use sync::Mutex;
     use sys_util::{EventFd, GuestMemory};
 
     struct DummyDevice(u32);
@@ -260,6 +263,7 @@ mod tests {
             _mem: GuestMemory,
             _interrupt_evt: EventFd,
             _interrupt_resample_evt: EventFd,
+            _msix_config: Option<Arc<Mutex<MsixConfig>>>,
             _status: Arc<AtomicUsize>,
             _queues: Vec<Queue>,
             _queue_evts: Vec<EventFd>,
@@ -278,6 +282,7 @@ mod tests {
             device_feature_select: 0x0,
             driver_feature_select: 0x0,
             queue_select: 0xff,
+            msix_config: 0x00,
         };
 
         let dev = &mut DummyDevice(0) as &mut dyn VirtioDevice;
diff --git a/devices/src/virtio/virtio_pci_device.rs b/devices/src/virtio/virtio_pci_device.rs
index 33b6a5e..2110b75 100644
--- a/devices/src/virtio/virtio_pci_device.rs
+++ b/devices/src/virtio/virtio_pci_device.rs
@@ -6,6 +6,7 @@ use std;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
+use sync::Mutex;
 
 use data_model::{DataInit, Le32};
 use kvm::Datamatch;
@@ -14,8 +15,8 @@ use sys_util::{EventFd, GuestMemory, Result};
 
 use super::*;
 use crate::pci::{
-    MsixCap, PciBarConfiguration, PciCapability, PciCapabilityID, PciClassCode, PciConfiguration,
-    PciDevice, PciDeviceError, PciHeaderType, PciInterruptPin, PciSubclass,
+    MsixCap, MsixConfig, PciBarConfiguration, PciCapability, PciCapabilityID, PciClassCode,
+    PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciInterruptPin, PciSubclass,
 };
 
 use self::virtio_pci_common_config::VirtioPciCommonConfig;
@@ -137,7 +138,9 @@ const DEVICE_CONFIG_SIZE: u64 = 0x1000;
 const NOTIFICATION_BAR_OFFSET: u64 = 0x3000;
 const NOTIFICATION_SIZE: u64 = 0x1000;
 const MSIX_TABLE_BAR_OFFSET: u64 = 0x6000;
+const MSIX_TABLE_SIZE: u64 = 0x1000;
 const MSIX_PBA_BAR_OFFSET: u64 = 0x7000;
+const MSIX_PBA_SIZE: u64 = 0x1000;
 const CAPABILITY_BAR_SIZE: u64 = 0x8000;
 
 const NOTIFY_OFF_MULTIPLIER: u32 = 4; // A dword per notification address.
@@ -162,6 +165,7 @@ pub struct VirtioPciDevice {
     queue_evts: Vec<EventFd>,
     mem: Option<GuestMemory>,
     settings_bar: u8,
+    msix_config: Option<Arc<Mutex<MsixConfig>>>,
     msix_cap_reg_idx: Option<usize>,
     common_config: VirtioPciCommonConfig,
 }
@@ -181,6 +185,14 @@ impl VirtioPciDevice {
 
         let pci_device_id = VIRTIO_PCI_DEVICE_ID_BASE + device.device_type() as u16;
 
+        let msix_num = device.msix_vectors();
+        let msix_config = if msix_num > 0 {
+            let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_num)));
+            Some(msix_config)
+        } else {
+            None
+        };
+
         let config_regs = PciConfiguration::new(
             VIRTIO_PCI_VENDOR_ID,
             pci_device_id,
@@ -204,6 +216,7 @@ impl VirtioPciDevice {
             queue_evts,
             mem: Some(mem),
             settings_bar: 0,
+            msix_config,
             msix_cap_reg_idx: None,
             common_config: VirtioPciCommonConfig {
                 driver_status: 0,
@@ -211,6 +224,7 @@ impl VirtioPciDevice {
                 device_feature_select: 0,
                 driver_feature_select: 0,
                 queue_select: 0,
+                msix_config: 0,
             },
         })
     }
@@ -295,7 +309,7 @@ impl VirtioPciDevice {
             .add_capability(&configuration_cap)
             .map_err(PciDeviceError::CapabilitiesSetup)?;
 
-        if self.device.msix_vectors() > 0 {
+        if self.msix_config.is_some() && self.device.msix_vectors() > 0 {
             let msix_cap = MsixCap::new(
                 settings_bar,
                 self.device.msix_vectors(),
@@ -447,10 +461,27 @@ impl PciDevice for VirtioPciDevice {
     }
 
     fn read_config_register(&self, reg_idx: usize) -> u32 {
-        self.config_regs.read_reg(reg_idx)
+        let mut data: u32 = self.config_regs.read_reg(reg_idx);
+        if let Some(msix_cap_reg_idx) = self.msix_cap_reg_idx {
+            if let Some(msix_config) = &self.msix_config {
+                if msix_cap_reg_idx == reg_idx {
+                    data = msix_config.lock().read_msix_capability(data);
+                }
+            }
+        }
+
+        data
     }
 
     fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+        if let Some(msix_cap_reg_idx) = self.msix_cap_reg_idx {
+            if let Some(msix_config) = &self.msix_config {
+                if msix_cap_reg_idx == reg_idx {
+                    msix_config.lock().write_msix_capability(offset, data);
+                }
+            }
+        }
+
         (&mut self.config_regs).write_reg(reg_idx, offset, data)
     }
 
@@ -489,6 +520,23 @@ impl PciDevice for VirtioPciDevice {
             {
                 // Handled with ioeventfds.
             }
+
+            o if MSIX_TABLE_BAR_OFFSET <= o && o < MSIX_TABLE_BAR_OFFSET + MSIX_TABLE_SIZE => {
+                if let Some(msix_config) = &self.msix_config {
+                    msix_config
+                        .lock()
+                        .read_msix_table(o - MSIX_TABLE_BAR_OFFSET, data);
+                }
+            }
+
+            o if MSIX_PBA_BAR_OFFSET <= o && o < MSIX_PBA_BAR_OFFSET + MSIX_PBA_SIZE => {
+                if let Some(msix_config) = &self.msix_config {
+                    msix_config
+                        .lock()
+                        .read_pba_entries(o - MSIX_PBA_BAR_OFFSET, data);
+                }
+            }
+
             _ => (),
         }
     }
@@ -524,6 +572,21 @@ impl PciDevice for VirtioPciDevice {
             {
                 // Handled with ioeventfds.
             }
+            o if MSIX_TABLE_BAR_OFFSET <= o && o < MSIX_TABLE_BAR_OFFSET + MSIX_TABLE_SIZE => {
+                if let Some(msix_config) = &self.msix_config {
+                    msix_config
+                        .lock()
+                        .write_msix_table(o - MSIX_TABLE_BAR_OFFSET, data);
+                }
+            }
+            o if MSIX_PBA_BAR_OFFSET <= o && o < MSIX_PBA_BAR_OFFSET + MSIX_PBA_SIZE => {
+                if let Some(msix_config) = &self.msix_config {
+                    msix_config
+                        .lock()
+                        .write_pba_entries(o - MSIX_PBA_BAR_OFFSET, data);
+                }
+            }
+
             _ => (),
         };
 
@@ -531,10 +594,17 @@ impl PciDevice for VirtioPciDevice {
             if let Some(interrupt_evt) = self.interrupt_evt.take() {
                 if let Some(interrupt_resample_evt) = self.interrupt_resample_evt.take() {
                     if let Some(mem) = self.mem.take() {
+                        let msix_config = if let Some(msix_config) = &self.msix_config {
+                            Some(msix_config.clone())
+                        } else {
+                            None
+                        };
+
                         self.device.activate(
                             mem,
                             interrupt_evt,
                             interrupt_resample_evt,
+                            msix_config,
                             self.interrupt_status.clone(),
                             self.queues.clone(),
                             self.queue_evts.split_off(0),
diff --git a/devices/src/virtio/wl.rs b/devices/src/virtio/wl.rs
index a4988cb..4531511 100644
--- a/devices/src/virtio/wl.rs
+++ b/devices/src/virtio/wl.rs
@@ -48,6 +48,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
+use sync::Mutex;
 
 #[cfg(feature = "wl-dmabuf")]
 use libc::{dup, EBADF, EINVAL};
@@ -73,6 +74,7 @@ use super::resource_bridge::*;
 use super::{
     DescriptorChain, Queue, VirtioDevice, INTERRUPT_STATUS_USED_RING, TYPE_WL, VIRTIO_F_VERSION_1,
 };
+use crate::pci::MsixConfig;
 use vm_control::{MaybeOwnedFd, VmMemoryControlRequestSocket, VmMemoryRequest, VmMemoryResponse};
 
 const VIRTWL_SEND_MAX_ALLOCS: usize = 28;
@@ -1743,6 +1745,7 @@ impl VirtioDevice for Wl {
         mem: GuestMemory,
         interrupt_evt: EventFd,
         interrupt_resample_evt: EventFd,
+        _msix_config: Option<Arc<Mutex<MsixConfig>>>,
         status: Arc<AtomicUsize>,
         mut queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,