vhost-net: implement direct msix irq fd

The current vhost-net msix irq injection flow is from vhost-kernel
to crosvm vhost-net, then to the KVM for irq injection. It still need
crosvm vhost-net to trigger irq, which is because the set_vring_call
is not directly using the msix irq fd.

To optimize this flow to be from vhost-kernel to KVM directly, need:
1. if the msix is enabled and unmasked, use the misx irq fd for the
vring_call directly so that all the misx injection can directly to
KVM from vhost-kernel.
2. if the msix is disabled or masked, use the indirect vhost_interrupt
fd to let the crosvm to control the irq injection.

BUG=None
TEST=cargo test -p devices
TEST=start crosvm with vhost-net, and run the iperf3 on the network
without any issue

Change-Id: Idb3427f69f23b728305ed63d88973156a03e7c6b
Signed-off-by: Chuanxiao Dong <chuanxiao.dong@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2046452
Reviewed-by: Stephen Barber <smbarber@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>

2 files changed, 35 insertions, 9 deletions

diff --git a/devices/src/pci/mod.rs b/devices/src/pci/mod.rs
index 8c5380e..b9ebc19 100644
--- a/devices/src/pci/mod.rs
+++ b/devices/src/pci/mod.rs
@@ -15,7 +15,7 @@ mod pci_root;
 mod vfio_pci;
 
 pub use self::ac97::{Ac97Backend, Ac97Dev, Ac97Parameters};
-pub use self::msix::{MsixCap, MsixConfig};
+pub use self::msix::{MsixCap, MsixConfig, MsixStatus};
 pub use self::pci_configuration::{
     PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityID,
     PciClassCode, PciConfiguration, PciHeaderType, PciProgrammingInterface, PciSerialBusSubClass,
diff --git a/devices/src/pci/msix.rs b/devices/src/pci/msix.rs
index 6c79b4a..b69114b 100644
--- a/devices/src/pci/msix.rs
+++ b/devices/src/pci/msix.rs
@@ -89,6 +89,12 @@ impl Display for MsixError {
 
 type MsixResult<T> = std::result::Result<T, MsixError>;
 
+pub enum MsixStatus {
+    Changed,
+    EntryChanged(usize),
+    NothingToDo,
+}
+
 impl MsixConfig {
     pub fn new(msix_vectors: u16, vm_socket: Arc<VmIrqRequestSocket>) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
@@ -123,6 +129,18 @@ impl MsixConfig {
         self.masked
     }
 
+    /// Check whether the Function Mask bit in MSIX table Message Control
+    /// word in set or not.
+    /// If true, the vector is masked.
+    /// If false, the vector is unmasked.
+    pub fn table_masked(&self, index: usize) -> bool {
+        if index >= self.table_entries.len() {
+            true
+        } else {
+            self.table_entries[index].masked()
+        }
+    }
+
     /// Check whether the MSI-X Enable bit in Message Control word in set or not.
     /// if 1, the function is permitted to use MSI-X to request service.
     pub fn enabled(&self) -> bool {
@@ -147,7 +165,7 @@ impl MsixConfig {
 
     /// Write to the MSI-X Capability Structure.
     /// Only the top 2 bits in Message Control Word are writable.
-    pub fn write_msix_capability(&mut self, offset: u64, data: &[u8]) {
+    pub fn write_msix_capability(&mut self, offset: u64, data: &[u8]) -> MsixStatus {
         if offset == 2 && data.len() == 2 {
             let reg = u16::from_le_bytes([data[0], data[1]]);
             let old_masked = self.masked;
@@ -173,6 +191,9 @@ impl MsixConfig {
                         self.inject_msix_and_clear_pba(index);
                     }
                 }
+                return MsixStatus::Changed;
+            } else if !old_masked && self.masked {
+                return MsixStatus::Changed;
             }
         } else {
             error!(
@@ -180,6 +201,7 @@ impl MsixConfig {
                 offset
             );
         }
+        MsixStatus::NothingToDo
     }
 
     fn add_msi_route(&self, index: u16, gsi: u32) -> MsixResult<()> {
@@ -304,7 +326,7 @@ impl MsixConfig {
     /// Vector Control: only bit 0 (Mask Bit) is not reserved: when this bit
     ///     is set, the function is prohibited from sending a message using
     ///     this MSI-X Table entry.
-    pub fn write_msix_table(&mut self, offset: u64, data: &[u8]) {
+    pub fn write_msix_table(&mut self, offset: u64, data: &[u8]) -> MsixStatus {
         let index: usize = (offset / MSIX_TABLE_ENTRIES_MODULO) as usize;
         let modulo_offset = offset % MSIX_TABLE_ENTRIES_MODULO;
 
@@ -360,13 +382,17 @@ impl MsixConfig {
         // device.
 
         // Check if bit has been flipped
-        if !self.masked()
-            && old_entry.masked()
-            && !self.table_entries[index].masked()
-            && self.get_pba_bit(index as u16) == 1
-        {
-            self.inject_msix_and_clear_pba(index);
+        if !self.masked() {
+            if old_entry.masked() && !self.table_entries[index].masked() {
+                if self.get_pba_bit(index as u16) == 1 {
+                    self.inject_msix_and_clear_pba(index);
+                }
+                return MsixStatus::EntryChanged(index);
+            } else if !old_entry.masked() && self.table_entries[index].masked() {
+                return MsixStatus::EntryChanged(index);
+            }
         }
+        MsixStatus::NothingToDo
     }
 
     /// Read PBA Entries