path: root/pkgs/cloud-hypervisor/0002-virtio-devices-add-a-GPU-device.patch

From 4dbb7b2a0ffe3ac55c1c0a0dde992597cd2959e3 Mon Sep 17 00:00:00 2001
From: Alyssa Ross <alyssa.ross@unikie.com>
Date: Wed, 7 Sep 2022 14:16:29 +0000
Subject: [PATCH 2/2] virtio-devices: add a GPU device
SPDX-FileCopyrightText: The Cloud Hypervisor Authors
SPDX-FileCopyrightText: 2017-2018 The Chromium OS Authors. All rights reserved.
SPDX-FileCopyrightText: 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
SPDX-FileCopyrightText: 2019-2020 Intel Corporation
SPDX-FileCopyrightText: 2020, Oracle and/or its affiliates.
SPDX-FileCopyrightText: 2022 Unikie
SPDX-FileCopyrightText: 2023-2024 Alyssa Ross <hi@alyssa.is>
SPDX-License-Identifier: Apache-2.0 AND LicenseRef-BSD-3-Clause-Google

This adds support for exposing a virtio-gpu device to guest by
implementing a vhost-user frontend compatible with crosvm's GPU device
backend.

Note that this is not the same as the "vhost-user-gpu" protocol
implemented by QEMU.

Adding a GPU device from the command line looks like this:

    --gpu socket=/path/to/crosvm-gpu-vhost-user.sock

Signed-off-by: Alyssa Ross <alyssa.ross@unikie.com>
Co-authored-by: Alyssa Ross <hi@alyssa.is>
Signed-off-by: Alyssa Ross <hi@alyssa.is>
---
 src/bin/ch-remote.rs                          |  44 ++
 src/main.rs                                   |   8 +
 virtio-devices/src/device.rs                  |   4 +-
 virtio-devices/src/lib.rs                     |   4 +-
 virtio-devices/src/seccomp_filters.rs         |  16 +
 virtio-devices/src/transport/pci_device.rs    |   4 +-
 virtio-devices/src/vhost_user/gpu.rs          | 415 ++++++++++++++++++
 virtio-devices/src/vhost_user/mod.rs          |   6 +
 .../src/vhost_user/vu_common_ctrl.rs          |   9 +-
 vmm/src/api/dbus/mod.rs                       |   7 +-
 vmm/src/api/http/http_endpoint.rs             |   9 +-
 vmm/src/api/http/mod.rs                       |  10 +-
 vmm/src/api/mod.rs                            |  47 +-
 vmm/src/api/openapi/cloud-hypervisor.yaml     |  39 ++
 vmm/src/config.rs                             | 108 +++++
 vmm/src/device_manager.rs                     | 135 +++++-
 vmm/src/lib.rs                                |  79 +++-
 vmm/src/vm.rs                                 |  28 +-
 vmm/src/vm_config.rs                          |  10 +
 19 files changed, 960 insertions(+), 22 deletions(-)
 create mode 100644 virtio-devices/src/vhost_user/gpu.rs

diff --git a/src/bin/ch-remote.rs b/src/bin/ch-remote.rs
index 1d5d5d0b..2d528f07 100644
--- a/src/bin/ch-remote.rs
+++ b/src/bin/ch-remote.rs
@@ -30,6 +30,7 @@ enum Error {
     AddDeviceConfig(vmm::config::Error),
     AddDiskConfig(vmm::config::Error),
     AddFsConfig(vmm::config::Error),
+    AddGpuConfig(vmm::config::Error),
     AddPmemConfig(vmm::config::Error),
     AddNetConfig(vmm::config::Error),
     AddUserDeviceConfig(vmm::config::Error),
@@ -53,6 +54,7 @@ impl fmt::Display for Error {
             AddDeviceConfig(e) => write!(f, "Error parsing device syntax: {e}"),
             AddDiskConfig(e) => write!(f, "Error parsing disk syntax: {e}"),
             AddFsConfig(e) => write!(f, "Error parsing filesystem syntax: {e}"),
+            AddGpuConfig(e) => write!(f, "Error parsing GPU syntax: {e}"),
             AddPmemConfig(e) => write!(f, "Error parsing persistent memory syntax: {e}"),
             AddNetConfig(e) => write!(f, "Error parsing network syntax: {e}"),
             AddUserDeviceConfig(e) => write!(f, "Error parsing user device syntax: {e}"),
@@ -79,6 +81,7 @@ trait DBusApi1 {
     fn vm_add_device(&self, device_config: &str) -> zbus::Result<Optional<String>>;
     fn vm_add_disk(&self, disk_config: &str) -> zbus::Result<Optional<String>>;
     fn vm_add_fs(&self, fs_config: &str) -> zbus::Result<Optional<String>>;
+    fn vm_add_gpu(&self, gpu_config: &str) -> zbus::Result<Optional<String>>;
     fn vm_add_net(&self, net_config: &str) -> zbus::Result<Optional<String>>;
     fn vm_add_pmem(&self, pmem_config: &str) -> zbus::Result<Optional<String>>;
     fn vm_add_user_device(&self, vm_add_user_device: &str) -> zbus::Result<Optional<String>>;
@@ -151,6 +154,10 @@ impl<'a> DBusApi1ProxyBlocking<'a> {
         self.print_response(self.vm_add_fs(fs_config))
     }
 
+    fn api_vm_add_gpu(&self, gpu_config: &str) -> ApiResult {
+        self.print_response(self.vm_add_gpu(gpu_config))
+    }
+
     fn api_vm_add_net(&self, net_config: &str) -> ApiResult {
         self.print_response(self.vm_add_net(net_config))
     }
@@ -377,6 +384,17 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu
             simple_api_command(socket, "PUT", "add-fs", Some(&fs_config))
                 .map_err(Error::HttpApiClient)
         }
+        Some("add-gpu") => {
+            let gpu_config = add_gpu_config(
+                matches
+                    .subcommand_matches("add-gpu")
+                    .unwrap()
+                    .get_one::<String>("gpu_config")
+                    .unwrap(),
+            )?;
+            simple_api_command(socket, "PUT", "add-gpu", Some(&gpu_config))
+                .map_err(Error::HttpApiClient)
+        }
         Some("add-pmem") => {
             let pmem_config = add_pmem_config(
                 matches
@@ -599,6 +617,16 @@ fn dbus_api_do_command(matches: &ArgMatches, proxy: &DBusApi1ProxyBlocking<'_>)
             )?;
             proxy.api_vm_add_fs(&fs_config)
         }
+        Some("add-gpu") => {
+            let gpu_config = add_gpu_config(
+                matches
+                    .subcommand_matches("add-gpu")
+                    .unwrap()
+                    .get_one::<String>("gpu_config")
+                    .unwrap(),
+            )?;
+            proxy.api_vm_add_gpu(&gpu_config)
+        }
         Some("add-pmem") => {
             let pmem_config = add_pmem_config(
                 matches
@@ -806,6 +834,13 @@ fn add_fs_config(config: &str) -> Result<String, Error> {
     Ok(fs_config)
 }
 
+fn add_gpu_config(config: &str) -> Result<String, Error> {
+    let gpu_config = vmm::config::GpuConfig::parse(config).map_err(Error::AddGpuConfig)?;
+    let gpu_config = serde_json::to_string(&gpu_config).unwrap();
+
+    Ok(gpu_config)
+}
+
 fn add_pmem_config(config: &str) -> Result<String, Error> {
     let pmem_config = vmm::config::PmemConfig::parse(config).map_err(Error::AddPmemConfig)?;
     let pmem_config = serde_json::to_string(&pmem_config).unwrap();
@@ -943,6 +978,15 @@ fn main() {
                         .help(vmm::config::FsConfig::SYNTAX),
                 ),
         )
+        .subcommand(
+            Command::new("add-gpu")
+                .about("Add virtio-gpu backed gpu device")
+                .arg(
+                    Arg::new("gpu_config")
+                        .index(1)
+                        .help(vmm::config::GpuConfig::SYNTAX),
+                ),
+        )
         .subcommand(
             Command::new("add-pmem")
                 .about("Add persistent memory device")
diff --git a/src/main.rs b/src/main.rs
index 86d80562..149b4c74 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -285,6 +285,13 @@ fn create_app(default_vcpus: String, default_memory: String, default_rng: String
                 .num_args(1..)
                 .group("vm-config"),
         )
+        .arg(
+            Arg::new("gpu")
+                .long("gpu")
+                .help(config::GpuConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
         .arg(
             Arg::new("pmem")
                 .long("pmem")
@@ -885,6 +892,7 @@ mod unit_tests {
             },
             balloon: None,
             fs: None,
+            gpu: None,
             pmem: None,
             serial: ConsoleConfig {
                 file: None,
diff --git a/virtio-devices/src/device.rs b/virtio-devices/src/device.rs
index b70092f8..4b6e6a3b 100644
--- a/virtio-devices/src/device.rs
+++ b/virtio-devices/src/device.rs
@@ -11,7 +11,7 @@ use crate::{
     VIRTIO_F_RING_INDIRECT_DESC,
 };
 use libc::EFD_NONBLOCK;
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::io::Write;
 use std::num::Wrapping;
 use std::sync::{
@@ -59,7 +59,7 @@ pub struct VirtioSharedMemoryList {
     pub mem_slot: u32,
     pub addr: GuestAddress,
     pub len: GuestUsize,
-    pub region_list: Vec<VirtioSharedMemory>,
+    pub region_list: BTreeMap<u8, VirtioSharedMemory>,
 }
 
 /// Trait for virtio devices to be driven by a virtio transport.
diff --git a/virtio-devices/src/lib.rs b/virtio-devices/src/lib.rs
index 680cbe29..55428cd8 100644
--- a/virtio-devices/src/lib.rs
+++ b/virtio-devices/src/lib.rs
@@ -44,7 +44,7 @@ pub use self::block::{Block, BlockState};
 pub use self::console::{Console, ConsoleResizer, Endpoint};
 pub use self::device::{
     DmaRemapping, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioInterrupt,
-    VirtioInterruptType, VirtioSharedMemoryList,
+    VirtioInterruptType, VirtioSharedMemory, VirtioSharedMemoryList,
 };
 pub use self::epoll_helper::{
     EpollHelper, EpollHelperError, EpollHelperHandler, EPOLL_HELPER_EVENT_LAST,
@@ -93,6 +93,8 @@ pub enum ActivateError {
     VhostUserFsSetup(vhost_user::Error),
     #[error("Failed to setup vhost-user daemon: {0}")]
     VhostUserSetup(vhost_user::Error),
+    #[error("Failed to setup vhost-user-gpu daemon: {0}")]
+    VhostUserGpuSetup(vhost_user::Error),
     #[error("Failed to create seccomp filter: {0}")]
     CreateSeccompFilter(seccompiler::Error),
     #[error("Failed to create rate limiter: {0}")]
diff --git a/virtio-devices/src/seccomp_filters.rs b/virtio-devices/src/seccomp_filters.rs
index b9e7e5f1..5d35ae5b 100644
--- a/virtio-devices/src/seccomp_filters.rs
+++ b/virtio-devices/src/seccomp_filters.rs
@@ -22,6 +22,7 @@ pub enum Thread {
     VirtioRng,
     VirtioVhostBlock,
     VirtioVhostFs,
+    VirtioVhostGpu,
     VirtioVhostNet,
     VirtioVhostNetCtl,
     VirtioVsock,
@@ -164,6 +165,20 @@ fn virtio_vhost_fs_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
     ]
 }
 
+fn virtio_vhost_gpu_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
+    vec![
+        (libc::SYS_clock_nanosleep, vec![]),
+        (libc::SYS_connect, vec![]),
+        (libc::SYS_getcwd, vec![]),
+        (libc::SYS_nanosleep, vec![]),
+        (libc::SYS_recvmsg, vec![]),
+        (libc::SYS_recvmsg, vec![]),
+        (libc::SYS_sendmsg, vec![]),
+        (libc::SYS_sendmsg, vec![]),
+        (libc::SYS_socket, vec![]),
+    ]
+}
+
 fn virtio_vhost_net_ctl_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
     vec![]
 }
@@ -234,6 +249,7 @@ fn get_seccomp_rules(thread_type: Thread) -> Vec<(i64, Vec<SeccompRule>)> {
         Thread::VirtioRng => virtio_rng_thread_rules(),
         Thread::VirtioVhostBlock => virtio_vhost_block_thread_rules(),
         Thread::VirtioVhostFs => virtio_vhost_fs_thread_rules(),
+        Thread::VirtioVhostGpu => virtio_vhost_gpu_thread_rules(),
         Thread::VirtioVhostNet => virtio_vhost_net_thread_rules(),
         Thread::VirtioVhostNetCtl => virtio_vhost_net_ctl_thread_rules(),
         Thread::VirtioVsock => virtio_vsock_thread_rules(),
diff --git a/virtio-devices/src/transport/pci_device.rs b/virtio-devices/src/transport/pci_device.rs
index 2f711c80..246b6497 100644
--- a/virtio-devices/src/transport/pci_device.rs
+++ b/virtio-devices/src/transport/pci_device.rs
@@ -1053,11 +1053,11 @@ impl PciDevice for VirtioPciDevice {
                     PciDeviceError::IoRegistrationFailed(shm_list.addr.raw_value(), e)
                 })?;
 
-                for (idx, shm) in shm_list.region_list.iter().enumerate() {
+                for (shmid, shm) in shm_list.region_list.iter() {
                     let shm_cap = VirtioPciCap64::new(
                         PciCapabilityType::SharedMemory,
                         VIRTIO_SHM_BAR_INDEX as u8,
-                        idx as u8,
+                        *shmid,
                         shm.offset,
                         shm.len,
                     );
diff --git a/virtio-devices/src/vhost_user/gpu.rs b/virtio-devices/src/vhost_user/gpu.rs
new file mode 100644
index 00000000..0dd06f73
--- /dev/null
+++ b/virtio-devices/src/vhost_user/gpu.rs
@@ -0,0 +1,415 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// Copyright 2022 Unikie
+// Copyright 2023 Alyssa Ross <hi@alyssa.is>
+// SPDX-License-Identifier: Apache-2.0
+
+use super::vu_common_ctrl::VhostUserHandle;
+use super::{Error, Result};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
+use crate::vhost_user::VhostUserCommon;
+use crate::{
+    ActivateError, ActivateResult, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioDeviceType,
+    VirtioInterrupt, VirtioSharedMemoryList, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
+};
+use crate::{GuestMemoryMmap, GuestRegionMmap, MmapRegion};
+use seccompiler::SeccompAction;
+use std::io::{self, Write};
+use std::os::unix::io::AsRawFd;
+use std::result;
+use std::sync::{Arc, Barrier, Mutex};
+use std::thread;
+use vhost::vhost_user::message::{
+    VhostSharedMemoryRegion, VhostUserConfigFlags, VhostUserProtocolFeatures, VhostUserShmemMapMsg,
+    VhostUserShmemUnmapMsg, VhostUserVirtioFeatures,
+};
+use vhost::vhost_user::{
+    FrontendReqHandler, HandlerResult, VhostUserFrontend, VhostUserFrontendReqHandler,
+};
+use virtio_bindings::virtio_gpu::{
+    VIRTIO_GPU_F_CONTEXT_INIT, VIRTIO_GPU_F_RESOURCE_BLOB, VIRTIO_GPU_F_RESOURCE_UUID,
+    VIRTIO_GPU_F_VIRGL,
+};
+use virtio_queue::Queue;
+use vm_memory::GuestMemoryAtomic;
+use vm_migration::{MigratableError, Pausable};
+use vmm_sys_util::eventfd::EventFd;
+
+const QUEUE_SIZES: &[u16] = &[256, 16];
+const NUM_QUEUES: u16 = QUEUE_SIZES.len() as _;
+
+struct BackendReqHandler {
+    cache_size: u64,
+    mmap_cache_addr: u64,
+}
+
+impl BackendReqHandler {
+    // Make sure request is within cache range
+    fn is_req_valid(&self, offset: u64, len: u64) -> bool {
+        let end = match offset.checked_add(len) {
+            Some(n) => n,
+            None => return false,
+        };
+
+        !(offset >= self.cache_size || end > self.cache_size)
+    }
+}
+
+impl VhostUserFrontendReqHandler for BackendReqHandler {
+    fn shmem_map(&self, req: &VhostUserShmemMapMsg, fd: &dyn AsRawFd) -> HandlerResult<u64> {
+        if !self.is_req_valid(req.shm_offset, req.len) {
+            return Err(io::Error::from_raw_os_error(libc::EINVAL));
+        }
+
+        let addr = self.mmap_cache_addr + req.shm_offset;
+        let ret = unsafe {
+            libc::mmap(
+                addr as *mut libc::c_void,
+                req.len as usize,
+                req.flags.bits() as i32,
+                // https://bugzilla.kernel.org/show_bug.cgi?id=217238
+                if req.flags.bits() as i32 & libc::PROT_WRITE != 0 {
+                    libc::MAP_SHARED
+                } else {
+                    libc::MAP_PRIVATE
+                } | libc::MAP_FIXED,
+                fd.as_raw_fd(),
+                req.fd_offset as libc::off_t,
+            )
+        };
+
+        if ret == libc::MAP_FAILED {
+            return Err(io::Error::last_os_error());
+        }
+
+        Ok(0)
+    }
+
+    fn shmem_unmap(&self, req: &VhostUserShmemUnmapMsg) -> HandlerResult<u64> {
+        if !self.is_req_valid(req.shm_offset, req.len) {
+            return Err(io::Error::from_raw_os_error(libc::EINVAL));
+        }
+
+        let addr = self.mmap_cache_addr + req.shm_offset;
+        let ret = unsafe {
+            libc::mmap(
+                addr as *mut libc::c_void,
+                req.len as usize,
+                libc::PROT_NONE,
+                libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED,
+                -1,
+                0,
+            )
+        };
+        if ret == libc::MAP_FAILED {
+            return Err(io::Error::last_os_error());
+        }
+
+        Ok(0)
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct VirtioGpuConfig {
+    pub events_read: u32,
+    pub events_clear: u32,
+    pub num_scanouts: u32,
+    pub num_capsets: u32,
+}
+
+pub struct Gpu {
+    common: VirtioCommon,
+    vu_common: VhostUserCommon,
+    id: String,
+    // Hold ownership of the memory that is allocated for the device
+    // which will be automatically dropped when the device is dropped
+    cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
+    backend_req_support: bool,
+    seccomp_action: SeccompAction,
+    guest_memory: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
+    epoll_thread: Option<thread::JoinHandle<()>>,
+    exit_evt: EventFd,
+    iommu: bool,
+}
+
+impl Gpu {
+    /// Create a new virtio-gpu device.
+    pub fn new(
+        id: String,
+        path: &str,
+        seccomp_action: SeccompAction,
+        exit_evt: EventFd,
+        iommu: bool,
+    ) -> Result<(Gpu, VhostSharedMemoryRegion)> {
+        // Connect to the vhost-user socket.
+        let mut vu = VhostUserHandle::connect_vhost_user(false, path, NUM_QUEUES as u64, false)?;
+
+        let avail_features = 1 << VIRTIO_F_VERSION_1
+            | 1 << VIRTIO_GPU_F_VIRGL
+            | 1 << VIRTIO_GPU_F_RESOURCE_UUID
+            | 1 << VIRTIO_GPU_F_RESOURCE_BLOB
+            | 1 << VIRTIO_GPU_F_CONTEXT_INIT
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+        let avail_protocol_features = VhostUserProtocolFeatures::CONFIG
+            | VhostUserProtocolFeatures::BACKEND_REQ
+            | VhostUserProtocolFeatures::SHARED_MEMORY_REGIONS;
+
+        let (acked_features, acked_protocol_features) =
+            vu.negotiate_features_vhost_user(avail_features, avail_protocol_features)?;
+
+        let shm_regions = vu.get_shared_memory_regions()?;
+        if shm_regions.len() != 1 {
+            return Err(Error::VhostUserUnexpectedSharedMemoryRegionsCount(
+                1,
+                shm_regions.len(),
+            ));
+        }
+        let shm_region = shm_regions[0];
+
+        Ok((
+            Gpu {
+                common: VirtioCommon {
+                    device_type: VirtioDeviceType::Gpu as u32,
+                    avail_features: acked_features,
+                    // If part of the available features that have been acked, the
+                    // PROTOCOL_FEATURES bit must be already set through the VIRTIO
+                    // acked features as we know the guest would never ack it, this
+                    // the feature would be lost.
+                    acked_features: acked_features
+                        & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(),
+                    paused_sync: Some(Arc::new(Barrier::new(NUM_QUEUES as usize))),
+                    queue_sizes: QUEUE_SIZES.to_vec(),
+                    min_queues: NUM_QUEUES,
+                    ..Default::default()
+                },
+                vu_common: VhostUserCommon {
+                    vu: Some(Arc::new(Mutex::new(vu))),
+                    acked_protocol_features,
+                    socket_path: path.to_string(),
+                    vu_num_queues: NUM_QUEUES as usize,
+                    ..Default::default()
+                },
+                id,
+                cache: None,
+                backend_req_support: acked_protocol_features
+                    & VhostUserProtocolFeatures::BACKEND_REQ.bits()
+                    != 0,
+                seccomp_action,
+                guest_memory: None,
+                epoll_thread: None,
+                exit_evt,
+                iommu,
+            },
+            shm_region,
+        ))
+    }
+
+    pub fn set_cache(&mut self, cache: (VirtioSharedMemoryList, MmapRegion)) {
+        self.cache = Some(cache);
+    }
+}
+
+impl Drop for Gpu {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.common.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+    }
+}
+
+impl VirtioDevice for Gpu {
+    fn device_type(&self) -> u32 {
+        self.common.device_type
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.common.queue_sizes
+    }
+
+    fn features(&self) -> u64 {
+        let mut features = self.common.avail_features;
+        if self.iommu {
+            features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
+        features
+    }
+
+    fn ack_features(&mut self, value: u64) {
+        self.common.ack_features(value)
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        if let Some(vu) = &self.vu_common.vu {
+            if let Err(e) = vu
+                .lock()
+                .unwrap()
+                .socket_handle()
+                .get_config(
+                    offset as u32,
+                    data.len() as u32,
+                    VhostUserConfigFlags::WRITABLE,
+                    data,
+                )
+                .map_err(|e| format!("{:?}", e))
+                .and_then(|(_, config)| data.write_all(&config).map_err(|e| format!("{:?}", e)))
+            {
+                error!("Failed getting vhost-user-gpu configuration: {:?}", e);
+            }
+        }
+    }
+
+    fn activate(
+        &mut self,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        queues: Vec<(usize, Queue, EventFd)>,
+    ) -> ActivateResult {
+        self.common.activate(&queues, &interrupt_cb)?;
+        self.guest_memory = Some(mem.clone());
+
+        // Initialize backend communication.
+        let backend_req_handler = if self.backend_req_support {
+            if let Some(cache) = self.cache.as_ref() {
+                let vu_frontend_req_handler = Arc::new(BackendReqHandler {
+                    cache_size: cache.0.len,
+                    mmap_cache_addr: cache.0.host_addr,
+                });
+
+                let mut req_handler =
+                    FrontendReqHandler::new(vu_frontend_req_handler).map_err(|e| {
+                        ActivateError::VhostUserGpuSetup(Error::FrontendReqHandlerCreation(e))
+                    })?;
+
+                if self.vu_common.acked_protocol_features
+                    & VhostUserProtocolFeatures::REPLY_ACK.bits()
+                    != 0
+                {
+                    req_handler.set_reply_ack_flag(true);
+                }
+
+                Some(req_handler)
+            } else {
+                None
+            }
+        } else {
+            None
+        };
+
+        // Run a dedicated thread for handling potential reconnections with
+        // the backend.
+        let (kill_evt, pause_evt) = self.common.dup_eventfds();
+
+        let mut handler = self.vu_common.activate(
+            mem,
+            queues,
+            interrupt_cb,
+            self.common.acked_features,
+            backend_req_handler,
+            kill_evt,
+            pause_evt,
+        )?;
+
+        let paused = self.common.paused.clone();
+        let paused_sync = self.common.paused_sync.clone();
+
+        let mut epoll_threads = Vec::new();
+        spawn_virtio_thread(
+            &self.id,
+            &self.seccomp_action,
+            Thread::VirtioVhostGpu,
+            &mut epoll_threads,
+            &self.exit_evt,
+            move || handler.run(paused, paused_sync.unwrap()),
+        )?;
+        self.epoll_thread = Some(epoll_threads.remove(0));
+
+        event!("virtio-device", "activated", "id", &self.id);
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
+        // We first must resume the virtio thread if it was paused.
+        if self.common.pause_evt.take().is_some() {
+            self.common.resume().ok()?;
+        }
+
+        if let Some(vu) = &self.vu_common.vu {
+            if let Err(e) = vu.lock().unwrap().reset_vhost_user() {
+                error!("Failed to reset vhost-user daemon: {:?}", e);
+                return None;
+            }
+        }
+
+        if let Some(kill_evt) = self.common.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        event!("virtio-device", "reset", "id", &self.id);
+
+        // Return the interrupt
+        Some(self.common.interrupt_cb.take().unwrap())
+    }
+
+    fn shutdown(&mut self) {
+        self.vu_common.shutdown()
+    }
+
+    fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
+        self.cache.as_ref().map(|cache| cache.0.clone())
+    }
+
+    fn set_shm_regions(
+        &mut self,
+        shm_regions: VirtioSharedMemoryList,
+    ) -> std::result::Result<(), crate::Error> {
+        if let Some(cache) = self.cache.as_mut() {
+            cache.0 = shm_regions;
+            Ok(())
+        } else {
+            Err(crate::Error::SetShmRegionsNotSupported)
+        }
+    }
+
+    fn add_memory_region(
+        &mut self,
+        region: &Arc<GuestRegionMmap>,
+    ) -> std::result::Result<(), crate::Error> {
+        self.vu_common.add_memory_region(&self.guest_memory, region)
+    }
+
+    fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
+        let mut mappings = Vec::new();
+        if let Some(cache) = self.cache.as_ref() {
+            mappings.push(UserspaceMapping {
+                host_addr: cache.0.host_addr,
+                mem_slot: cache.0.mem_slot,
+                addr: cache.0.addr,
+                len: cache.0.len,
+                mergeable: false,
+            })
+        }
+
+        mappings
+    }
+}
+
+impl Pausable for Gpu {
+    fn pause(&mut self) -> result::Result<(), MigratableError> {
+        self.vu_common.pause()?;
+        self.common.pause()
+    }
+
+    fn resume(&mut self) -> result::Result<(), MigratableError> {
+        self.common.resume()?;
+
+        if let Some(epoll_thread) = &self.epoll_thread {
+            epoll_thread.thread().unpark();
+        }
+
+        self.vu_common.resume()
+    }
+}
diff --git a/virtio-devices/src/vhost_user/mod.rs b/virtio-devices/src/vhost_user/mod.rs
index a4f56c31..fe10b144 100644
--- a/virtio-devices/src/vhost_user/mod.rs
+++ b/virtio-devices/src/vhost_user/mod.rs
@@ -31,11 +31,13 @@ use vu_common_ctrl::VhostUserHandle;
 
 pub mod blk;
 pub mod fs;
+pub mod gpu;
 pub mod net;
 pub mod vu_common_ctrl;
 
 pub use self::blk::Blk;
 pub use self::fs::*;
+pub use self::gpu::*;
 pub use self::net::Net;
 pub use self::vu_common_ctrl::VhostUserConfig;
 
@@ -73,6 +75,8 @@ pub enum Error {
     VhostUserGetQueueMaxNum(VhostError),
     #[error("Get protocol features failed: {0}")]
     VhostUserGetProtocolFeatures(VhostError),
+    #[error("Get shared memory regions failed: {0}")]
+    VhostUserGetSharedMemoryRegions(VhostError),
     #[error("Get vring base failed: {0}")]
     VhostUserGetVringBase(VhostError),
     #[error("Vhost-user Backend not support vhost-user protocol")]
@@ -121,6 +125,8 @@ pub enum Error {
     VhostUserSetInflight(VhostError),
     #[error("Failed setting the log base: {0}")]
     VhostUserSetLogBase(VhostError),
+    #[error("Expected {0} shared memory regions; got {1}")]
+    VhostUserUnexpectedSharedMemoryRegionsCount(usize, usize),
     #[error("Invalid used address")]
     UsedAddress,
     #[error("Invalid features provided from vhost-user backend")]
diff --git a/virtio-devices/src/vhost_user/vu_common_ctrl.rs b/virtio-devices/src/vhost_user/vu_common_ctrl.rs
index 6f980f23..54511c52 100644
--- a/virtio-devices/src/vhost_user/vu_common_ctrl.rs
+++ b/virtio-devices/src/vhost_user/vu_common_ctrl.rs
@@ -18,7 +18,8 @@ use std::time::{Duration, Instant};
 use std::vec::Vec;
 use vhost::vhost_kern::vhost_binding::{VHOST_F_LOG_ALL, VHOST_VRING_F_LOG};
 use vhost::vhost_user::message::{
-    VhostUserHeaderFlag, VhostUserInflight, VhostUserProtocolFeatures, VhostUserVirtioFeatures,
+    VhostSharedMemoryRegion, VhostUserHeaderFlag, VhostUserInflight, VhostUserProtocolFeatures,
+    VhostUserVirtioFeatures,
 };
 use vhost::vhost_user::{
     Frontend, FrontendReqHandler, VhostUserFrontend, VhostUserFrontendReqHandler,
@@ -104,6 +105,12 @@ impl VhostUserHandle {
             .map_err(Error::VhostUserAddMemReg)
     }
 
+    pub fn get_shared_memory_regions(&self) -> Result<Vec<VhostSharedMemoryRegion>> {
+        self.vu
+            .get_shared_memory_regions()
+            .map_err(Error::VhostUserGetSharedMemoryRegions)
+    }
+
     pub fn negotiate_features_vhost_user(
         &mut self,
         avail_features: u64,
diff --git a/vmm/src/api/dbus/mod.rs b/vmm/src/api/dbus/mod.rs
index f8a6b64c..7bd86412 100644
--- a/vmm/src/api/dbus/mod.rs
+++ b/vmm/src/api/dbus/mod.rs
@@ -6,7 +6,7 @@ use super::{ApiAction, ApiRequest};
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
 use crate::api::VmCoredump;
 use crate::api::{
-    AddDisk, Body, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa,
+    AddDisk, Body, VmAddDevice, VmAddFs, VmAddGpu, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa,
     VmAddVsock, VmBoot, VmCounters, VmCreate, VmDelete, VmInfo, VmPause, VmPowerButton, VmReboot,
     VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore, VmResume,
     VmSendMigration, VmShutdown, VmSnapshot, VmmPing, VmmShutdown,
@@ -141,6 +141,11 @@ impl DBusApi {
         self.vm_action(&VmAddFs, fs_config).await
     }
 
+    async fn vm_add_gpu(&self, gpu_config: String) -> Result<Optional<String>> {
+        let gpu_config = serde_json::from_str(&gpu_config).map_err(api_error)?;
+        self.vm_action(&VmAddGpu, gpu_config).await
+    }
+
     async fn vm_add_net(&self, net_config: String) -> Result<Optional<String>> {
         let mut net_config: NetConfig = serde_json::from_str(&net_config).map_err(api_error)?;
         if net_config.fds.is_some() {
diff --git a/vmm/src/api/http/http_endpoint.rs b/vmm/src/api/http/http_endpoint.rs
index 141cc1d0..1cc8786b 100644
--- a/vmm/src/api/http/http_endpoint.rs
+++ b/vmm/src/api/http/http_endpoint.rs
@@ -8,10 +8,10 @@ use crate::api::http::{error_response, EndpointHandler, HttpError};
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
 use crate::api::VmCoredump;
 use crate::api::{
-    AddDisk, ApiAction, ApiRequest, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, VmAddUserDevice,
-    VmAddVdpa, VmAddVsock, VmBoot, VmConfig, VmCounters, VmDelete, VmPause, VmPowerButton,
-    VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore, VmResume,
-    VmSendMigration, VmShutdown, VmSnapshot,
+    AddDisk, ApiAction, ApiRequest, VmAddDevice, VmAddFs, VmAddGpu, VmAddNet, VmAddPmem,
+    VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmConfig, VmCounters, VmDelete, VmPause,
+    VmPowerButton, VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore,
+    VmResume, VmSendMigration, VmShutdown, VmSnapshot,
 };
 use crate::config::NetConfig;
 use micro_http::{Body, Method, Request, Response, StatusCode, Version};
@@ -176,6 +176,7 @@ vm_action_put_handler!(VmPowerButton);
 vm_action_put_handler_body!(VmAddDevice);
 vm_action_put_handler_body!(AddDisk);
 vm_action_put_handler_body!(VmAddFs);
+vm_action_put_handler_body!(VmAddGpu);
 vm_action_put_handler_body!(VmAddPmem);
 vm_action_put_handler_body!(VmAddVdpa);
 vm_action_put_handler_body!(VmAddVsock);
diff --git a/vmm/src/api/http/mod.rs b/vmm/src/api/http/mod.rs
index 2eac530f..8a15906f 100644
--- a/vmm/src/api/http/mod.rs
+++ b/vmm/src/api/http/mod.rs
@@ -7,9 +7,9 @@ use self::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdow
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
 use crate::api::VmCoredump;
 use crate::api::{
-    AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, VmAddUserDevice,
-    VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmPause, VmPowerButton, VmReboot,
-    VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore, VmResume,
+    AddDisk, ApiError, ApiRequest, VmAddDevice, VmAddFs, VmAddGpu, VmAddNet, VmAddPmem,
+    VmAddUserDevice, VmAddVdpa, VmAddVsock, VmBoot, VmCounters, VmDelete, VmPause, VmPowerButton,
+    VmReboot, VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore, VmResume,
     VmSendMigration, VmShutdown, VmSnapshot,
 };
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
@@ -176,6 +176,10 @@ pub static HTTP_ROUTES: Lazy<HttpRoutes> = Lazy::new(|| {
         endpoint!("/vm.add-fs"),
         Box::new(VmActionHandler::new(&VmAddFs)),
     );
+    r.routes.insert(
+        endpoint!("/vm.add-gpu"),
+        Box::new(VmActionHandler::new(&VmAddGpu)),
+    );
     r.routes.insert(
         endpoint!("/vm.add-net"),
         Box::new(VmActionHandler::new(&VmAddNet)),
diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs
index ad74aa9e..142bc870 100644
--- a/vmm/src/api/mod.rs
+++ b/vmm/src/api/mod.rs
@@ -39,8 +39,8 @@ pub use self::http::start_http_fd_thread;
 pub use self::http::start_http_path_thread;
 
 use crate::config::{
-    DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, UserDeviceConfig,
-    VdpaConfig, VmConfig, VsockConfig,
+    DeviceConfig, DiskConfig, FsConfig, GpuConfig, NetConfig, PmemConfig, RestoreConfig,
+    UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig,
 };
 use crate::device_tree::DeviceTree;
 use crate::vm::{Error as VmError, VmState};
@@ -139,6 +139,9 @@ pub enum ApiError {
     /// The fs could not be added to the VM.
     VmAddFs(VmError),
 
+    /// The gpu could not be added to the VM.
+    VmAddGpu(VmError),
+
     /// The pmem device could not be added to the VM.
     VmAddPmem(VmError),
 
@@ -193,6 +196,7 @@ impl Display for ApiError {
             ApplySeccompFilter(seccomp_error) => write!(f, "{}", seccomp_error),
             VmAddDisk(vm_error) => write!(f, "{}", vm_error),
             VmAddFs(vm_error) => write!(f, "{}", vm_error),
+            VmAddGpu(vm_error) => write!(f, "{}", vm_error),
             VmAddPmem(vm_error) => write!(f, "{}", vm_error),
             VmAddNet(vm_error) => write!(f, "{}", vm_error),
             VmAddVdpa(vm_error) => write!(f, "{}", vm_error),
@@ -332,6 +336,8 @@ pub trait RequestHandler {
 
     fn vm_add_fs(&mut self, fs_cfg: FsConfig) -> Result<Option<Vec<u8>>, VmError>;
 
+    fn vm_add_gpu(&mut self, gpu_cfg: GpuConfig) -> Result<Option<Vec<u8>>, VmError>;
+
     fn vm_add_pmem(&mut self, pmem_cfg: PmemConfig) -> Result<Option<Vec<u8>>, VmError>;
 
     fn vm_add_net(&mut self, net_cfg: NetConfig) -> Result<Option<Vec<u8>>, VmError>;
@@ -527,6 +533,43 @@ impl ApiAction for VmAddFs {
     }
 }
 
+pub struct VmAddGpu;
+
+impl ApiAction for VmAddGpu {
+    type RequestBody = GpuConfig;
+    type ResponseBody = Option<Body>;
+
+    fn request(
+        &self,
+        config: Self::RequestBody,
+        response_sender: Sender<ApiResponse>,
+    ) -> ApiRequest {
+        Box::new(move |vmm| {
+            info!("API request event: VmAddGpu {:?}", config);
+
+            let response = vmm
+                .vm_add_gpu(config)
+                .map_err(ApiError::VmAddGpu)
+                .map(ApiResponsePayload::VmAction);
+
+            response_sender
+                .send(response)
+                .map_err(VmmError::ApiResponseSend)?;
+
+            Ok(false)
+        })
+    }
+
+    fn send(
+        &self,
+        api_evt: EventFd,
+        api_sender: Sender<ApiRequest>,
+        data: Self::RequestBody,
+    ) -> ApiResult<Self::ResponseBody> {
+        get_response_body(self, api_evt, api_sender, data)
+    }
+}
+
 pub struct VmAddPmem;
 
 impl ApiAction for VmAddPmem {
diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml
index e8269771..3c5b90af 100644
--- a/vmm/src/api/openapi/cloud-hypervisor.yaml
+++ b/vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -259,6 +259,28 @@ paths:
         500:
           description: The new device could not be added to the VM instance.
 
+  /vm.add-gpu:
+    put:
+      summary: Add a new virtio-gpu device to the VM
+      requestBody:
+        description: The details of the new virtio-gpu
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/GpuConfig"
+        required: true
+      responses:
+        200:
+          description: The new device was successfully added to the VM instance.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/PciDeviceInfo"
+        204:
+          description: The new device was successfully (cold) added to the VM instance.
+        500:
+          description: The new device could not be added to the VM instance.
+
   /vm.add-pmem:
     put:
       summary: Add a new pmem device to the VM
@@ -574,6 +596,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/FsConfig"
+        gpu:
+          type: array
+          items:
+            $ref: "#/components/schemas/GpuConfig"
         pmem:
           type: array
           items:
@@ -980,6 +1006,19 @@ components:
         id:
           type: string
 
+    GpuConfig:
+      required:
+        - socket
+      type: object
+      properties:
+        socket:
+          type: string
+        pci_segment:
+          type: integer
+          format: int16
+        id:
+          type: string
+
     PmemConfig:
       required:
         - file
diff --git a/vmm/src/config.rs b/vmm/src/config.rs
index 4b4e618a..d21c9655 100644
--- a/vmm/src/config.rs
+++ b/vmm/src/config.rs
@@ -27,6 +27,8 @@ pub enum Error {
     ParseFsTagMissing,
     /// Filesystem socket is missing
     ParseFsSockMissing,
+    /// GPU socket is missing
+    ParseGpuSockMissing,
     /// Missing persistent memory file parameter.
     ParsePmemFileMissing,
     /// Missing vsock socket path parameter.
@@ -57,6 +59,8 @@ pub enum Error {
     ParseBalloon(OptionParserError),
     /// Error parsing filesystem parameters
     ParseFileSystem(OptionParserError),
+    /// Error parsing GPU parameters
+    ParseGpu(OptionParserError),
     /// Error parsing persistent memory parameters
     ParsePersistentMemory(OptionParserError),
     /// Failed parsing console
@@ -350,6 +354,8 @@ impl fmt::Display for Error {
             ParseFileSystem(o) => write!(f, "Error parsing --fs: {o}"),
             ParseFsSockMissing => write!(f, "Error parsing --fs: socket missing"),
             ParseFsTagMissing => write!(f, "Error parsing --fs: tag missing"),
+            ParseGpu(o) => write!(f, "Error parsing --gpu: {o}"),
+            ParseGpuSockMissing => write!(f, "Error parsing --gpu: socket missing"),
             ParsePersistentMemory(o) => write!(f, "Error parsing --pmem: {o}"),
             ParsePmemFileMissing => write!(f, "Error parsing --pmem: file missing"),
             ParseVsock(o) => write!(f, "Error parsing --vsock: {o}"),
@@ -416,6 +422,7 @@ pub struct VmParams<'a> {
     pub rng: &'a str,
     pub balloon: Option<&'a str>,
     pub fs: Option<Vec<&'a str>>,
+    pub gpu: Option<Vec<&'a str>>,
     pub pmem: Option<Vec<&'a str>>,
     pub serial: &'a str,
     pub console: &'a str,
@@ -468,6 +475,9 @@ impl<'a> VmParams<'a> {
         let fs: Option<Vec<&str>> = args
             .get_many::<String>("fs")
             .map(|x| x.map(|y| y as &str).collect());
+        let gpu: Option<Vec<&str>> = args
+            .get_many::<String>("gpu")
+            .map(|x| x.map(|y| y as &str).collect());
         let pmem: Option<Vec<&str>> = args
             .get_many::<String>("pmem")
             .map(|x| x.map(|y| y as &str).collect());
@@ -510,6 +520,7 @@ impl<'a> VmParams<'a> {
             rng,
             balloon,
             fs,
+            gpu,
             pmem,
             serial,
             console,
@@ -1559,6 +1570,49 @@ impl FsConfig {
     }
 }
 
+impl GpuConfig {
+    pub const SYNTAX: &'static str = "virtio-gpu parameters \
+    \"socket=<socket_path>,id=<device_id>,pci_segment=<segment_id>\"";
+
+    pub fn parse(gpu: &str) -> Result<Self> {
+        let mut parser = OptionParser::new();
+        parser.add("socket").add("id").add("pci_segment");
+        parser.parse(gpu).map_err(Error::ParseGpu)?;
+
+        let socket = PathBuf::from(parser.get("socket").ok_or(Error::ParseGpuSockMissing)?);
+        let id = parser.get("id");
+
+        let pci_segment = parser
+            .convert("pci_segment")
+            .map_err(Error::ParseGpu)?
+            .unwrap_or_default();
+
+        Ok(GpuConfig {
+            socket,
+            id,
+            pci_segment,
+        })
+    }
+
+    pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> {
+        if let Some(platform_config) = vm_config.platform.as_ref() {
+            if self.pci_segment >= platform_config.num_pci_segments {
+                return Err(ValidationError::InvalidPciSegment(self.pci_segment));
+            }
+
+            if let Some(iommu_segments) = platform_config.iommu_segments.as_ref() {
+                if iommu_segments.contains(&self.pci_segment) {
+                    return Err(ValidationError::IommuNotSupportedOnSegment(
+                        self.pci_segment,
+                    ));
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
 impl PmemConfig {
     pub const SYNTAX: &'static str = "Persistent memory parameters \
     \"file=<backing_file_path>,size=<persistent_memory_size>,iommu=on|off,\
@@ -2237,6 +2291,17 @@ impl VmConfig {
             }
         }
 
+        if let Some(gpus) = &self.gpu {
+            if !gpus.is_empty() && !self.memory.shared {
+                return Err(ValidationError::VhostUserRequiresSharedMemory);
+            }
+            for gpu in gpus {
+                gpu.validate(self)?;
+
+                Self::validate_identifier(&mut id_list, &gpu.id)?;
+            }
+        }
+
         if let Some(pmems) = &self.pmem {
             for pmem in pmems {
                 pmem.validate(self)?;
@@ -2467,6 +2532,15 @@ impl VmConfig {
             fs = Some(fs_config_list);
         }
 
+        let mut gpu: Option<Vec<GpuConfig>> = None;
+        if let Some(gpu_list) = &vm_params.gpu {
+            let mut gpu_config_list = Vec::new();
+            for item in gpu_list.iter() {
+                gpu_config_list.push(GpuConfig::parse(item)?);
+            }
+            gpu = Some(gpu_config_list);
+        }
+
         let mut pmem: Option<Vec<PmemConfig>> = None;
         if let Some(pmem_list) = &vm_params.pmem {
             let mut pmem_config_list = Vec::new();
@@ -2585,6 +2659,7 @@ impl VmConfig {
             rng,
             balloon,
             fs,
+            gpu,
             pmem,
             serial,
             console,
@@ -2641,6 +2716,13 @@ impl VmConfig {
             removed |= fs.len() != len;
         }
 
+        // Remove if gpu device
+        if let Some(gpu) = self.gpu.as_mut() {
+            let len = gpu.len();
+            gpu.retain(|dev| dev.id.as_ref().map(|id| id.as_ref()) != Some(id));
+            removed |= gpu.len() != len;
+        }
+
         // Remove if net device
         if let Some(net) = self.net.as_mut() {
             let len = net.len();
@@ -2711,6 +2793,7 @@ impl Clone for VmConfig {
             rng: self.rng.clone(),
             balloon: self.balloon.clone(),
             fs: self.fs.clone(),
+            gpu: self.gpu.clone(),
             pmem: self.pmem.clone(),
             serial: self.serial.clone(),
             console: self.console.clone(),
@@ -3177,6 +3260,23 @@ mod tests {
         Ok(())
     }
 
+    fn gpu_fixture() -> GpuConfig {
+        GpuConfig {
+            socket: PathBuf::from("/tmp/sock"),
+            id: None,
+            pci_segment: 0,
+        }
+    }
+
+    #[test]
+    fn test_parse_gpu() -> Result<()> {
+        // "socket" must be supplied
+        assert!(GpuConfig::parse("").is_err());
+        assert_eq!(GpuConfig::parse("socket=/tmp/sock")?, gpu_fixture());
+
+        Ok(())
+    }
+
     fn pmem_fixture() -> PmemConfig {
         PmemConfig {
             file: PathBuf::from("/tmp/pmem"),
@@ -3463,6 +3563,7 @@ mod tests {
             },
             balloon: None,
             fs: None,
+            gpu: None,
             pmem: None,
             serial: ConsoleConfig {
                 file: None,
@@ -3630,6 +3731,13 @@ mod tests {
             Err(ValidationError::VhostUserRequiresSharedMemory)
         );
 
+        let mut invalid_config = valid_config.clone();
+        invalid_config.gpu = Some(vec![gpu_fixture()]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::VhostUserRequiresSharedMemory)
+        );
+
         let mut still_valid_config = valid_config.clone();
         still_valid_config.memory.shared = true;
         assert!(still_valid_config.validate().is_ok());
diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs
index 6c8e5bc9..460ee543 100644
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -10,8 +10,8 @@
 //
 
 use crate::config::{
-    ConsoleOutputMode, DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, UserDeviceConfig,
-    VdpaConfig, VhostMode, VmConfig, VsockConfig,
+    ConsoleOutputMode, DeviceConfig, DiskConfig, FsConfig, GpuConfig, NetConfig, PmemConfig,
+    UserDeviceConfig, VdpaConfig, VhostMode, VmConfig, VsockConfig,
 };
 use crate::cpu::{CpuManager, CPU_MANAGER_ACPI_SIZE};
 use crate::device_tree::{DeviceNode, DeviceTree};
@@ -67,6 +67,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::fs::{read_link, File, OpenOptions};
 use std::io::{self, stdout, Seek, SeekFrom};
+use std::iter::once;
 use std::mem::zeroed;
 use std::num::Wrapping;
 use std::os::unix::fs::OpenOptionsExt;
@@ -82,6 +83,7 @@ use virtio_devices::transport::{VirtioPciDevice, VirtioPciDeviceActivator};
 use virtio_devices::vhost_user::VhostUserConfig;
 use virtio_devices::{
     AccessPlatformMapping, ActivateError, VdpaDmaMapping, VirtioMemMappingSource,
+    VirtioSharedMemory, VirtioSharedMemoryList,
 };
 use virtio_devices::{Endpoint, IommuMapping};
 use vm_allocator::{AddressAllocator, SystemAllocator};
@@ -127,6 +129,7 @@ const PVPANIC_DEVICE_NAME: &str = "__pvpanic";
 // identifiers if the user doesn't give one
 const DISK_DEVICE_NAME_PREFIX: &str = "_disk";
 const FS_DEVICE_NAME_PREFIX: &str = "_fs";
+const GPU_DEVICE_NAME_PREFIX: &str = "_gpu";
 const NET_DEVICE_NAME_PREFIX: &str = "_net";
 const PMEM_DEVICE_NAME_PREFIX: &str = "_pmem";
 const VDPA_DEVICE_NAME_PREFIX: &str = "_vdpa";
@@ -163,9 +166,15 @@ pub enum DeviceManagerError {
     /// Cannot create virtio-fs device
     CreateVirtioFs(virtio_devices::vhost_user::Error),
 
+    /// Cannot create virtio-gpu device
+    CreateVirtioGpu(virtio_devices::vhost_user::Error),
+
     /// Virtio-fs device was created without a socket.
     NoVirtioFsSock,
 
+    /// Virtio-gpu device was created without a socket.
+    NoVirtioGpuSock,
+
     /// Cannot create vhost-user-blk device
     CreateVhostUserBlk(virtio_devices::vhost_user::Error),
 
@@ -250,6 +259,9 @@ pub enum DeviceManagerError {
     /// Cannot find a memory range for virtio-fs
     FsRangeAllocation,
 
+    /// Cannot find a memory range for virtio-gpu
+    GpuRangeAllocation,
+
     /// Error creating serial output file
     SerialOutputFileOpen(io::Error),
 
@@ -2304,6 +2316,9 @@ impl DeviceManager {
         // Add virtio-fs if required
         devices.append(&mut self.make_virtio_fs_devices()?);
 
+        // Add virtio-gpu if required
+        devices.append(&mut self.make_virtio_gpu_devices()?);
+
         // Add virtio-pmem if required
         devices.append(&mut self.make_virtio_pmem_devices()?);
 
@@ -2875,6 +2890,114 @@ impl DeviceManager {
         Ok(devices)
     }
 
+    fn make_virtio_gpu_device(
+        &mut self,
+        gpu_cfg: &mut GpuConfig,
+    ) -> DeviceManagerResult<MetaVirtioDevice> {
+        let id = if let Some(id) = &gpu_cfg.id {
+            id.clone()
+        } else {
+            let id = self.next_device_name(GPU_DEVICE_NAME_PREFIX)?;
+            gpu_cfg.id = Some(id.clone());
+            id
+        };
+
+        info!("Creating virtio-gpu device: {:?}", gpu_cfg);
+
+        let mut node = device_node!(id);
+
+        if let Some(gpu_socket) = gpu_cfg.socket.to_str() {
+            let (mut virtio_gpu_device, region) = virtio_devices::vhost_user::Gpu::new(
+                id.clone(),
+                gpu_socket,
+                self.seccomp_action.clone(),
+                self.exit_evt
+                    .try_clone()
+                    .map_err(DeviceManagerError::EventFd)?,
+                self.force_iommu,
+            )
+            .map_err(DeviceManagerError::CreateVirtioGpu)?;
+
+            // In crosvm, the 8 GiB bar is 8 GiB-aligned.
+            let cache_base = self.pci_segments[gpu_cfg.pci_segment as usize]
+                .mem64_allocator
+                .lock()
+                .unwrap()
+                .allocate(None, region.length as GuestUsize, Some(region.length))
+                .ok_or(DeviceManagerError::GpuRangeAllocation)?
+                .raw_value();
+
+            // Update the node with correct resource information.
+            node.resources.push(Resource::MmioAddressRange {
+                base: cache_base,
+                size: region.length,
+            });
+
+            let mmap_region = MmapRegion::build(
+                None,
+                region.length as usize,
+                libc::PROT_NONE,
+                libc::MAP_ANONYMOUS | libc::MAP_PRIVATE,
+            )
+            .map_err(DeviceManagerError::NewMmapRegion)?;
+            let host_addr: u64 = mmap_region.as_ptr() as u64;
+
+            let mem_slot = self
+                .memory_manager
+                .lock()
+                .unwrap()
+                .create_userspace_mapping(cache_base, region.length, host_addr, false, false, false)
+                .map_err(DeviceManagerError::MemoryManager)?;
+
+            let region_list = once((
+                region.id,
+                VirtioSharedMemory {
+                    offset: 0,
+                    len: region.length,
+                },
+            ))
+            .collect();
+
+            virtio_gpu_device.set_cache((
+                VirtioSharedMemoryList {
+                    host_addr,
+                    mem_slot,
+                    addr: GuestAddress(cache_base),
+                    len: region.length as GuestUsize,
+                    region_list,
+                },
+                mmap_region,
+            ));
+
+            self.device_tree.lock().unwrap().insert(id.clone(), node);
+
+            Ok(MetaVirtioDevice {
+                virtio_device: Arc::new(Mutex::new(virtio_gpu_device))
+                    as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
+                iommu: false,
+                id,
+                pci_segment: gpu_cfg.pci_segment,
+                dma_handler: None,
+            })
+        } else {
+            Err(DeviceManagerError::NoVirtioGpuSock)
+        }
+    }
+
+    fn make_virtio_gpu_devices(&mut self) -> DeviceManagerResult<Vec<MetaVirtioDevice>> {
+        let mut devices = Vec::new();
+
+        let mut gpu_devices = self.config.lock().unwrap().gpu.clone();
+        if let Some(gpu_list_cfg) = &mut gpu_devices {
+            for gpu_cfg in gpu_list_cfg.iter_mut() {
+                devices.push(self.make_virtio_gpu_device(gpu_cfg)?);
+            }
+        }
+        self.config.lock().unwrap().gpu = gpu_devices;
+
+        Ok(devices)
+    }
+
     fn make_virtio_pmem_device(
         &mut self,
         pmem_cfg: &mut PmemConfig,
@@ -4138,6 +4261,7 @@ impl DeviceManager {
                 | VirtioDeviceType::Block
                 | VirtioDeviceType::Pmem
                 | VirtioDeviceType::Fs
+                | VirtioDeviceType::Gpu
                 | VirtioDeviceType::Vsock => {}
                 _ => return Err(DeviceManagerError::RemovalNotAllowed(device_type)),
             }
@@ -4410,6 +4534,13 @@ impl DeviceManager {
         self.hotplug_virtio_pci_device(device)
     }
 
+    pub fn add_gpu(&mut self, gpu_cfg: &mut GpuConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&gpu_cfg.id)?;
+
+        let device = self.make_virtio_gpu_device(gpu_cfg)?;
+        self.hotplug_virtio_pci_device(device)
+    }
+
     pub fn add_pmem(&mut self, pmem_cfg: &mut PmemConfig) -> DeviceManagerResult<PciDeviceInfo> {
         self.validate_identifier(&pmem_cfg.id)?;
 
diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs
index 121f14a5..951bc93b 100644
--- a/vmm/src/lib.rs
+++ b/vmm/src/lib.rs
@@ -13,8 +13,8 @@ use crate::api::{
     VmSendMigrationData, VmmPingResponse,
 };
 use crate::config::{
-    add_to_config, DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig,
-    UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig,
+    add_to_config, DeviceConfig, DiskConfig, FsConfig, GpuConfig, NetConfig, PmemConfig,
+    RestoreConfig, UserDeviceConfig, VdpaConfig, VmConfig, VsockConfig,
 };
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
 use crate::coredump::GuestDebuggable;
@@ -1704,6 +1704,32 @@ impl RequestHandler for Vmm {
         }
     }
 
+    fn vm_add_gpu(&mut self, gpu_cfg: GpuConfig) -> result::Result<Option<Vec<u8>>, VmError> {
+        self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?;
+
+        {
+            // Validate the configuration change in a cloned configuration
+            let mut config = self.vm_config.as_ref().unwrap().lock().unwrap().clone();
+            add_to_config(&mut config.gpu, gpu_cfg.clone());
+            config.validate().map_err(VmError::ConfigValidation)?;
+        }
+
+        if let Some(ref mut vm) = self.vm {
+            let info = vm.add_gpu(gpu_cfg).map_err(|e| {
+                error!("Error when adding new gpu to the VM: {:?}", e);
+                e
+            })?;
+            serde_json::to_vec(&info)
+                .map(Some)
+                .map_err(VmError::SerializeJson)
+        } else {
+            // Update VmConfig by adding the new device.
+            let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
+            add_to_config(&mut config.gpu, gpu_cfg);
+            Ok(None)
+        }
+    }
+
     fn vm_add_pmem(&mut self, pmem_cfg: PmemConfig) -> result::Result<Option<Vec<u8>>, VmError> {
         self.vm_config.as_ref().ok_or(VmError::VmNotCreated)?;
 
@@ -2094,6 +2120,7 @@ mod unit_tests {
             },
             balloon: None,
             fs: None,
+            gpu: None,
             pmem: None,
             serial: ConsoleConfig {
                 file: None,
@@ -2325,6 +2352,54 @@ mod unit_tests {
         );
     }
 
+    #[test]
+    fn test_vmm_vm_cold_add_gpu() {
+        let mut vmm = create_dummy_vmm();
+        let gpu_config = GpuConfig::parse("socket=/tmp/sock").unwrap();
+
+        assert!(matches!(
+            vmm.vm_add_gpu(gpu_config.clone()),
+            Err(VmError::VmNotCreated)
+        ));
+
+        let _ = vmm.vm_create(create_dummy_vm_config());
+        assert!(vmm
+            .vm_config
+            .as_ref()
+            .unwrap()
+            .lock()
+            .unwrap()
+            .gpu
+            .is_none());
+
+        let result = vmm.vm_add_gpu(gpu_config.clone());
+        assert!(result.is_ok());
+        assert!(result.unwrap().is_none());
+        assert_eq!(
+            vmm.vm_config
+                .as_ref()
+                .unwrap()
+                .lock()
+                .unwrap()
+                .gpu
+                .clone()
+                .unwrap()
+                .len(),
+            1
+        );
+        assert_eq!(
+            vmm.vm_config
+                .as_ref()
+                .unwrap()
+                .lock()
+                .unwrap()
+                .gpu
+                .clone()
+                .unwrap()[0],
+            gpu_config
+        );
+    }
+
     #[test]
     fn test_vmm_vm_cold_add_pmem() {
         let mut vmm = create_dummy_vmm();
diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs
index 43a04bfe..5cdd8fcf 100644
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -12,8 +12,8 @@
 //
 
 use crate::config::{
-    add_to_config, DeviceConfig, DiskConfig, FsConfig, HotplugMethod, NetConfig, PmemConfig,
-    UserDeviceConfig, ValidationError, VdpaConfig, VmConfig, VsockConfig,
+    add_to_config, DeviceConfig, DiskConfig, FsConfig, GpuConfig, HotplugMethod, NetConfig,
+    PmemConfig, UserDeviceConfig, ValidationError, VdpaConfig, VmConfig, VsockConfig,
 };
 use crate::config::{NumaConfig, PayloadConfig};
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
@@ -1582,6 +1582,30 @@ impl Vm {
         Ok(pci_device_info)
     }
 
+    pub fn add_gpu(&mut self, mut gpu_cfg: GpuConfig) -> Result<PciDeviceInfo> {
+        let pci_device_info = self
+            .device_manager
+            .lock()
+            .unwrap()
+            .add_gpu(&mut gpu_cfg)
+            .map_err(Error::DeviceManager)?;
+
+        // Update VmConfig by adding the new device. This is important to
+        // ensure the device would be created in case of a reboot.
+        {
+            let mut config = self.config.lock().unwrap();
+            add_to_config(&mut config.gpu, gpu_cfg);
+        }
+
+        self.device_manager
+            .lock()
+            .unwrap()
+            .notify_hotplug(AcpiNotificationFlags::PCI_DEVICES_CHANGED)
+            .map_err(Error::DeviceManager)?;
+
+        Ok(pci_device_info)
+    }
+
     pub fn add_pmem(&mut self, mut pmem_cfg: PmemConfig) -> Result<PciDeviceInfo> {
         let pci_device_info = self
             .device_manager
diff --git a/vmm/src/vm_config.rs b/vmm/src/vm_config.rs
index 51e7847e..afc5c72b 100644
--- a/vmm/src/vm_config.rs
+++ b/vmm/src/vm_config.rs
@@ -365,6 +365,15 @@ pub fn default_fsconfig_queue_size() -> u16 {
     1024
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+pub struct GpuConfig {
+    pub socket: PathBuf,
+    #[serde(default)]
+    pub id: Option<String>,
+    #[serde(default)]
+    pub pci_segment: u16,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct PmemConfig {
     pub file: PathBuf,
@@ -561,6 +570,7 @@ pub struct VmConfig {
     pub rng: RngConfig,
     pub balloon: Option<BalloonConfig>,
     pub fs: Option<Vec<FsConfig>>,
+    pub gpu: Option<Vec<GpuConfig>>,
     pub pmem: Option<Vec<PmemConfig>>,
     #[serde(default = "default_serial")]
     pub serial: ConsoleConfig,
-- 
2.43.0