Virtio-net: Let queue-num could configurable

Virtio-net multi queue feature need multi queue pairs, this patch change
queue-num configurable instead of static. --net-vq-pairs parameter could
config virtio net device virtual queue pairs.

BUG=chromium:1064482
TEST=Test virtio net function in guest with --net-vq-pairs=1 and
--net-vq-pairs=2

Change-Id: I75202c8ae52a83b8087b52149ec6d2138d8831cb
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2120312
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Xiong  Zhang <xiong.y.zhang@intel.corp-partner.google.com>
Reviewed-by: Stephen Barber <smbarber@chromium.org>

4 files changed, 51 insertions, 11 deletions

diff --git a/devices/src/virtio/net.rs b/devices/src/virtio/net.rs
index 0200daf..a15ab03 100644
--- a/devices/src/virtio/net.rs
+++ b/devices/src/virtio/net.rs
@@ -28,8 +28,6 @@ use super::{
 };
 
 const QUEUE_SIZE: u16 = 256;
-const NUM_QUEUES: usize = 3;
-const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE, QUEUE_SIZE, QUEUE_SIZE];
 
 #[derive(Debug)]
 pub enum NetError {
@@ -422,6 +420,7 @@ where
 }
 
 pub struct Net<T: TapT> {
+    queue_sizes: Box<[u16]>,
     workers_kill_evt: Vec<EventFd>,
     kill_evts: Vec<EventFd>,
     worker_threads: Vec<thread::JoinHandle<Worker<T>>>,
@@ -440,8 +439,8 @@ where
         ip_addr: Ipv4Addr,
         netmask: Ipv4Addr,
         mac_addr: MacAddress,
+        vq_pairs: u16,
     ) -> Result<Net<T>, NetError> {
-        let vq_pairs = QUEUE_SIZES.len() as u16 / 2;
         let multi_queue = if vq_pairs > 1 { true } else { false };
         let tap: T = T::new(true, multi_queue).map_err(NetError::TapOpen)?;
         tap.set_ip_addr(ip_addr).map_err(NetError::TapSetIp)?;
@@ -490,6 +489,7 @@ where
         }
 
         Ok(Net {
+            queue_sizes: vec![QUEUE_SIZE; (vq_pairs * 2 + 1) as usize].into_boxed_slice(),
             workers_kill_evt,
             kill_evts,
             worker_threads: Vec::new(),
@@ -500,7 +500,7 @@ where
     }
 
     fn build_config(&self) -> VirtioNetConfig {
-        let vq_pairs = QUEUE_SIZES.len() as u16 / 2;
+        let vq_pairs = self.queue_sizes.len() as u16 / 2;
 
         VirtioNetConfig {
             max_vq_pairs: Le16::from(vq_pairs),
@@ -600,7 +600,7 @@ where
     }
 
     fn queue_max_sizes(&self) -> &[u16] {
-        QUEUE_SIZES
+        &self.queue_sizes
     }
 
     fn features(&self) -> u64 {
@@ -643,12 +643,16 @@ where
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) {
-        if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
-            error!("net: expected {} queues, got {}", NUM_QUEUES, queues.len());
+        if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
+            error!(
+                "net: expected {} queues, got {}",
+                self.queue_sizes.len(),
+                queues.len()
+            );
             return;
         }
 
-        let vq_pairs = QUEUE_SIZES.len() / 2;
+        let vq_pairs = self.queue_sizes.len() / 2;
         if self.taps.len() != vq_pairs {
             error!("net: expected {} taps, got {}", vq_pairs, self.taps.len());
             return;
diff --git a/src/crosvm.rs b/src/crosvm.rs
index 81344c3..a55d2e4 100644
--- a/src/crosvm.rs
+++ b/src/crosvm.rs
@@ -176,6 +176,7 @@ pub struct Config {
     pub host_ip: Option<net::Ipv4Addr>,
     pub netmask: Option<net::Ipv4Addr>,
     pub mac_address: Option<net_util::MacAddress>,
+    pub net_vq_pairs: Option<u16>,
     pub vhost_net: bool,
     pub tap_fd: Vec<RawFd>,
     pub cid: Option<u64>,
@@ -223,6 +224,7 @@ impl Default for Config {
             host_ip: None,
             netmask: None,
             mac_address: None,
+            net_vq_pairs: None,
             vhost_net: false,
             tap_fd: Vec::new(),
             cid: None,
diff --git a/src/linux.rs b/src/linux.rs
index 6eb1ef7..0baf5ea 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -593,7 +593,13 @@ fn create_tap_net_device(cfg: &Config, tap_fd: RawFd) -> DeviceResult {
             .map_err(Error::CreateTapDevice)?
     };
 
-    let dev = virtio::Net::from(tap, 1).map_err(Error::NetDeviceNew)?;
+    let mut vq_pairs = cfg.net_vq_pairs.unwrap_or(1);
+    let vcpu_count = cfg.vcpu_count.unwrap_or(1);
+    if vcpu_count < vq_pairs as u32 {
+        error!("net vq pairs must be smaller than vcpu count, fall back to single queue mode");
+        vq_pairs = 1;
+    }
+    let dev = virtio::Net::from(tap, vq_pairs).map_err(Error::NetDeviceNew)?;
 
     Ok(VirtioDeviceStub {
         dev: Box::new(dev),
@@ -608,14 +614,21 @@ fn create_net_device(
     mac_address: MacAddress,
     mem: &GuestMemory,
 ) -> DeviceResult {
+    let mut vq_pairs = cfg.net_vq_pairs.unwrap_or(1);
+    let vcpu_count = cfg.vcpu_count.unwrap_or(1);
+    if vcpu_count < vq_pairs as u32 {
+        error!("net vq pairs must be smaller than vcpu count, fall back to single queue mode");
+        vq_pairs = 1;
+    }
+
     let dev = if cfg.vhost_net {
         let dev =
             virtio::vhost::Net::<Tap, vhost::Net<Tap>>::new(host_ip, netmask, mac_address, mem)
                 .map_err(Error::VhostNetDeviceNew)?;
         Box::new(dev) as Box<dyn VirtioDevice>
     } else {
-        let dev =
-            virtio::Net::<Tap>::new(host_ip, netmask, mac_address).map_err(Error::NetDeviceNew)?;
+        let dev = virtio::Net::<Tap>::new(host_ip, netmask, mac_address, vq_pairs)
+            .map_err(Error::NetDeviceNew)?;
         Box::new(dev) as Box<dyn VirtioDevice>
     };
 
diff --git a/src/main.rs b/src/main.rs
index 3afca8e..241334d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -762,6 +762,26 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
                         })?,
                 )
         }
+        "net-vq-pairs" => {
+            if cfg.net_vq_pairs.is_some() {
+                return Err(argument::Error::TooManyArguments(
+                    "`net-vq-pairs` already given".to_owned(),
+                ));
+            }
+            cfg.net_vq_pairs =
+                Some(
+                    value
+                        .unwrap()
+                        .parse()
+                        .map_err(|_| argument::Error::InvalidValue {
+                            value: value.unwrap().to_owned(),
+                            expected: String::from(
+                                "this value for `net-vq-pairs` needs to be integer",
+                            ),
+                        })?,
+                )
+        }
+
         "wayland-sock" => {
             let mut components = value.unwrap().split(',');
             let path =
@@ -1237,6 +1257,7 @@ fn run_vm(args: std::env::Args) -> std::result::Result<(), ()> {
                           "IP address to assign to host tap interface."),
           Argument::value("netmask", "NETMASK", "Netmask for VM subnet."),
           Argument::value("mac", "MAC", "MAC address for VM."),
+          Argument::value("net-vq-pairs", "N", "virtio net virtual queue paris. (default: 1)"),
           Argument::value("ac97",
                           "[backend=BACKEND,capture=true,capture_effect=EFFECT]",
                           "Comma separated key=value pairs for setting up Ac97 devices. Can be given more than once .