vfio: Integrate VFIO device into pci device model

Create VFIO device and VFIO PCI device in create_devices() function, and
intergrate it into PciRootBridge, so guest could see this vfio device.

Add a vfio config parameter, this config point to passthrough or mdev
device sysfs path.

For passthrough case, first user unbind host device from its driver,
then bind host device to vfio-pci. Like:
echo 0000:00:02.0 > /sys/bus/pci/devices/0000:00:02.0/driver/unbind
ech0 8086 1912 > /sys/bus/pci/drivers/vfio-pci/new_id
Finally pass the sysfs to crosvm through
--vfio=/sys/bus/pci/devices/0000:00:02.0

For mdev case, user create a mdev device through
echo $UUID > mdev_type/create, then pass this mdev device to crosvm like
--vfio=/sys/bus/pci/devices/0000:00:02.0/$UUID

BUG=chromium:992270
TEST=none

Change-Id: I0f59d6e93f62f9ab0727ad3a867d204f4ff6ad2d
Signed-off-by: Xiong Zhang <xiong.y.zhang@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1581140
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Reviewed-by: Zach Reizner <zachr@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>

3 files changed, 34 insertions, 1 deletions

diff --git a/src/crosvm.rs b/src/crosvm.rs
index bc39e7f..b7055df 100644
--- a/src/crosvm.rs
+++ b/src/crosvm.rs
@@ -109,6 +109,7 @@ pub struct Config {
     pub virtio_keyboard: Option<PathBuf>,
     pub virtio_input_evdevs: Vec<PathBuf>,
     pub split_irqchip: bool,
+    pub vfio: Option<PathBuf>,
 }
 
 impl Default for Config {
@@ -153,6 +154,7 @@ impl Default for Config {
             virtio_keyboard: None,
             virtio_input_evdevs: Vec::new(),
             split_irqchip: false,
+            vfio: None,
         }
     }
 }
diff --git a/src/linux.rs b/src/linux.rs
index 58e0971..f2433f7 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -27,7 +27,10 @@ use libc::{self, c_int, gid_t, uid_t};
 
 use audio_streams::DummyStreamSource;
 use devices::virtio::{self, VirtioDevice};
-use devices::{self, HostBackendDeviceProvider, PciDevice, VirtioPciDevice, XhciController};
+use devices::{
+    self, HostBackendDeviceProvider, PciDevice, VfioDevice, VfioPciDevice, VirtioPciDevice,
+    XhciController,
+};
 use io_jail::{self, Minijail};
 use kvm::*;
 use libcras::CrasClient;
@@ -91,6 +94,7 @@ pub enum Error {
     CreateTimerFd(sys_util::Error),
     CreateTpmStorage(PathBuf, io::Error),
     CreateUsbProvider(devices::usb::host_backend::error::Error),
+    CreateVfioDevice(devices::vfio::VfioError),
     DeviceJail(io_jail::Error),
     DevicePivotRoot(io_jail::Error),
     Disk(io::Error),
@@ -172,6 +176,7 @@ impl Display for Error {
                 write!(f, "failed to create tpm storage dir {}: {}", p.display(), e)
             }
             CreateUsbProvider(e) => write!(f, "failed to create usb provider: {}", e),
+            CreateVfioDevice(e) => write!(f, "Failed to create vfio device {}", e),
             DeviceJail(e) => write!(f, "failed to jail device: {}", e),
             DevicePivotRoot(e) => write!(f, "failed to pivot root device: {}", e),
             Disk(e) => write!(f, "failed to load disk image: {}", e),
@@ -979,6 +984,13 @@ fn create_devices(
     let usb_controller = Box::new(XhciController::new(mem.clone(), usb_provider));
     pci_devices.push((usb_controller, simple_jail(&cfg, "xhci.policy")?));
 
+    if cfg.vfio.is_some() {
+        let vfio_path = cfg.vfio.as_ref().unwrap().as_path();
+        let vfiodevice = Box::new(VfioDevice::new(vfio_path, vm).map_err(Error::CreateVfioDevice)?);
+        let vfiopcidevice = Box::new(VfioPciDevice::new(vfiodevice));
+        pci_devices.push((vfiopcidevice, simple_jail(&cfg, "vfio_device.policy")?));
+    }
+
     Ok(pci_devices)
 }
 
diff --git a/src/main.rs b/src/main.rs
index 75d08ae..a864833 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -685,6 +685,24 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
             }
             cfg.executable_path = Some(Executable::Bios(PathBuf::from(value.unwrap().to_owned())));
         }
+        "vfio" => {
+            let vfio_path = PathBuf::from(value.unwrap());
+            if !vfio_path.exists() {
+                return Err(argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: "the vfio path does not exist",
+                });
+            }
+            if !vfio_path.is_dir() {
+                return Err(argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: "the vfio path should be directory",
+                });
+            }
+
+            cfg.vfio = Some(vfio_path);
+        }
+
         "help" => return Err(argument::Error::PrintHelp),
         _ => unreachable!(),
     }
@@ -774,6 +792,7 @@ fn run_vm(args: std::env::Args) -> std::result::Result<(), ()> {
           #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
           Argument::flag("split-irqchip", "(EXPERIMENTAL) enable split-irqchip support"),
           Argument::value("bios", "PATH", "Path to BIOS/firmware ROM"),
+          Argument::value("vfio", "PATH", "Path to sysfs of pass through or mdev device"),
           Argument::short_flag('h', "help", "Print help message.")];
 
     let mut cfg = Config::default();