crosvm: add plugin API for async writes

A plugin might care to be immediately notified when a write
is made to a port, but it doesn't care to have the VM stopped
while the plugin calls back to resume the VM.

Unfortunately this means that multiple messages can be queued up in the
pipe and read() together by the plugin API.  Protobuf's parsing function
doesn't report how many bytes it read, so I've resorted to having crosvm
prefix every message with a length and then have the plugin lib parse
this number.  Impact on performance has not been measured.

BUG=b:143294496
TEST=Local build and run of build_test.  Verified that new unit
test was executed, exercised the case where multiple msgs are
received together, and completed successfully.

Change-Id: If6ef463e7b4d2e688e649f832a764fa644bf2d36
Signed-off-by: Matt Delco <delco@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1896376
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Stephen Barber <smbarber@chromium.org>

7 files changed, 460 insertions, 44 deletions

diff --git a/crosvm_plugin/crosvm.h b/crosvm_plugin/crosvm.h
index 700ab0c..63763f1 100644
--- a/crosvm_plugin/crosvm.h
+++ b/crosvm_plugin/crosvm.h
@@ -47,7 +47,7 @@ extern "C" {
  * do not indicate anything about what version of crosvm is running.
  */
 #define CROSVM_API_MAJOR 0
-#define CROSVM_API_MINOR 18
+#define CROSVM_API_MINOR 19
 #define CROSVM_API_PATCH 0
 
 enum crosvm_address_space {
@@ -175,6 +175,23 @@ int crosvm_reserve_range(struct crosvm*, uint32_t __space, uint64_t __start,
                          uint64_t __length);
 
 /*
+ * Registers a range in the given address space that, when accessed via write,
+ * will cause a notification in crosvm_vcpu_wait() but the VM will continue
+ * running.
+ * For this type of notification (where |no_resume| is set) the next call
+ * should be crosvm_vcpu_wait() (without an inbetween call to
+ * crosvm_vcpu_resume() ).
+ *
+ * The requested range must not overlap any prior (and currently active)
+ * reservation to crosvm_reserve_range() or crosvm_reserve_async_write_range().
+ *
+ * To unreserve a range previously reserved by this function, pass the |__space|
+ * and |__start| of the old reservation with a 0 |__length|.
+ */
+int crosvm_reserve_async_write_range(struct crosvm*, uint32_t __space,
+                                     uint64_t __start, uint64_t __length);
+
+/*
  * Sets the state of the given irq pin.
  */
 int crosvm_set_irq(struct crosvm*, uint32_t __irq_id, bool __active);
@@ -509,7 +526,13 @@ struct crosvm_vcpu_event {
        */
       uint8_t is_write;
 
-      uint8_t _reserved[3];
+      /*
+       * Valid when |is_write| is true -- indicates that VM has continued
+       * to run.  The only next valid call for the vcpu is crosvm_vcpu_wait().
+       */
+      uint8_t no_resume;
+
+      uint8_t _reserved[2];
     } io_access;
 
     /* CROSVM_VCPU_EVENT_KIND_PAUSED */
diff --git a/crosvm_plugin/src/lib.rs b/crosvm_plugin/src/lib.rs
index 8def28d..53a8569 100644
--- a/crosvm_plugin/src/lib.rs
+++ b/crosvm_plugin/src/lib.rs
@@ -171,6 +171,7 @@ pub enum Stat {
     GetMsrIndexList,
     NetGetConfig,
     ReserveRange,
+    ReserveAsyncWriteRange,
     SetIrq,
     SetIrqRouting,
     GetPicState,
@@ -466,12 +467,19 @@ impl crosvm {
         Ok(())
     }
 
-    fn reserve_range(&mut self, space: u32, start: u64, length: u64) -> result::Result<(), c_int> {
+    fn reserve_range(
+        &mut self,
+        space: u32,
+        start: u64,
+        length: u64,
+        async_write: bool,
+    ) -> result::Result<(), c_int> {
         let mut r = MainRequest::new();
         let reserve: &mut MainRequest_ReserveRange = r.mut_reserve_range();
         reserve.space = AddressSpace::from_i32(space as i32).ok_or(EINVAL)?;
         reserve.start = start;
         reserve.length = length;
+        reserve.async_write = async_write;
 
         self.main_transaction(&r, &[])?;
         Ok(())
@@ -910,7 +918,8 @@ struct anon_io_access {
     data: *mut u8,
     length: u32,
     is_write: u8,
-    __reserved1: u8,
+    no_resume: u8,
+    __reserved1: [u8; 2],
 }
 
 #[repr(C)]
@@ -949,6 +958,8 @@ pub struct crosvm_vcpu {
     send_init: bool,
     request_buffer: Vec<u8>,
     response_buffer: Vec<u8>,
+    response_base: usize,
+    response_length: usize,
     resume_data: Vec<u8>,
 
     regs: crosvm_vcpu_reg_cache,
@@ -956,6 +967,25 @@ pub struct crosvm_vcpu {
     debugregs: crosvm_vcpu_reg_cache,
 }
 
+fn read_varint32(data: &[u8]) -> (u32, usize) {
+    let mut value: u32 = 0;
+    let mut shift: u32 = 0;
+    for (i, &b) in data.iter().enumerate() {
+        if b < 0x80 {
+            return match (b as u32).checked_shl(shift) {
+                None => (0, 0),
+                Some(b) => (value | b, i + 1),
+            };
+        }
+        match ((b as u32) & 0x7F).checked_shl(shift) {
+            None => return (0, 0),
+            Some(b) => value |= b,
+        }
+        shift += 7;
+    }
+    (0, 0)
+}
+
 impl crosvm_vcpu {
     fn new(read_pipe: File, write_pipe: File) -> crosvm_vcpu {
         crosvm_vcpu {
@@ -964,6 +994,8 @@ impl crosvm_vcpu {
             send_init: true,
             request_buffer: Vec::new(),
             response_buffer: vec![0; MAX_DATAGRAM_SIZE],
+            response_base: 0,
+            response_length: 0,
             resume_data: Vec::new(),
             regs: crosvm_vcpu_reg_cache {
                 get: false,
@@ -994,13 +1026,30 @@ impl crosvm_vcpu {
     }
 
     fn vcpu_recv(&mut self) -> result::Result<VcpuResponse, c_int> {
-        let msg_size = self
-            .read_pipe
-            .read(&mut self.response_buffer)
-            .map_err(|e| -e.raw_os_error().unwrap_or(EINVAL))?;
-
-        let response: VcpuResponse =
-            parse_from_bytes(&self.response_buffer[..msg_size]).map_err(proto_error_to_int)?;
+        if self.response_length == 0 {
+            let msg_size = self
+                .read_pipe
+                .read(&mut self.response_buffer)
+                .map_err(|e| -e.raw_os_error().unwrap_or(EINVAL))?;
+            self.response_base = 0;
+            self.response_length = msg_size;
+        }
+        if self.response_length == 0 {
+            return Err(EINVAL);
+        }
+        let (value, bytes) = read_varint32(
+            &self.response_buffer[self.response_base..self.response_base + self.response_length],
+        );
+        let total_size: usize = bytes + value as usize;
+        if bytes == 0 || total_size > self.response_length {
+            return Err(EINVAL);
+        }
+        let response: VcpuResponse = parse_from_bytes(
+            &self.response_buffer[self.response_base + bytes..self.response_base + total_size],
+        )
+        .map_err(proto_error_to_int)?;
+        self.response_base += total_size;
+        self.response_length -= total_size;
         if response.errno != 0 {
             return Err(response.errno);
         }
@@ -1041,6 +1090,7 @@ impl crosvm_vcpu {
                 data: io.data.as_mut_ptr(),
                 length: io.data.len() as u32,
                 is_write: io.is_write as u8,
+                no_resume: io.no_resume as u8,
                 __reserved1: Default::default(),
             };
             self.resume_data = io.data;
@@ -1359,7 +1409,20 @@ pub unsafe extern "C" fn crosvm_reserve_range(
 ) -> c_int {
     let _u = record(Stat::ReserveRange);
     let self_ = &mut (*self_);
-    let ret = self_.reserve_range(space, start, length);
+    let ret = self_.reserve_range(space, start, length, false);
+    to_crosvm_rc(ret)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn crosvm_reserve_async_write_range(
+    self_: *mut crosvm,
+    space: u32,
+    start: u64,
+    length: u64,
+) -> c_int {
+    let _u = record(Stat::ReserveAsyncWriteRange);
+    let self_ = &mut (*self_);
+    let ret = self_.reserve_range(space, start, length, true);
     to_crosvm_rc(ret)
 }
 
diff --git a/protos/src/plugin.proto b/protos/src/plugin.proto
index 7b0600d..e2838b0 100644
--- a/protos/src/plugin.proto
+++ b/protos/src/plugin.proto
@@ -84,6 +84,7 @@ message MainRequest {
         AddressSpace space = 1;
         uint64 start = 2;
         uint64 length = 3;
+        bool async_write = 4;
     }
 
     message SetIrq {
@@ -384,12 +385,13 @@ message VcpuResponse  {
             AddressSpace space = 1;
             uint64 address = 2;
             bool is_write = 3;
-            bytes data = 4;
+            bool no_resume = 4;
+            bytes data = 5;
 
             // The following can be eagerly provided.
-            bytes regs = 5;
-            bytes sregs = 6;
-            bytes debugregs = 7;
+            bytes regs = 6;
+            bytes sregs = 7;
+            bytes debugregs = 8;
         }
 
         // This type of wait reason is only generated after a PuaseVcpus request on this VCPU.
diff --git a/src/plugin/process.rs b/src/plugin/process.rs
index 2869847..c2d6acb 100644
--- a/src/plugin/process.rs
+++ b/src/plugin/process.rs
@@ -380,7 +380,12 @@ impl Process {
                 };
                 match reserve_range.length {
                     0 => lock.unreserve_range(space, reserve_range.start),
-                    _ => lock.reserve_range(space, reserve_range.start, reserve_range.length),
+                    _ => lock.reserve_range(
+                        space,
+                        reserve_range.start,
+                        reserve_range.length,
+                        reserve_range.async_write,
+                    ),
                 }
             }
             Err(_) => Err(SysError::new(EDEADLK)),
diff --git a/src/plugin/vcpu.rs b/src/plugin/vcpu.rs
index 03d63b4..c623bda 100644
--- a/src/plugin/vcpu.rs
+++ b/src/plugin/vcpu.rs
@@ -23,6 +23,7 @@ use kvm_sys::{
     kvm_debugregs, kvm_fpu, kvm_lapic_state, kvm_mp_state, kvm_msr_entry, kvm_msrs, kvm_regs,
     kvm_sregs, kvm_vcpu_events, kvm_xcrs, KVM_CPUID_FLAG_SIGNIFCANT_INDEX,
 };
+use protobuf::stream::CodedOutputStream;
 use protos::plugin::*;
 use sync::Mutex;
 use sys_util::{error, LayoutAllocation};
@@ -37,7 +38,7 @@ pub enum IoSpace {
 }
 
 #[derive(Debug, Copy, Clone)]
-struct Range(u64, u64);
+struct Range(u64, u64, bool);
 
 impl Eq for Range {}
 
@@ -169,7 +170,13 @@ impl SharedVcpuState {
     /// Reserves the given range for handling by the plugin process.
     ///
     /// This will reject any reservation that overlaps with an existing reservation.
-    pub fn reserve_range(&mut self, space: IoSpace, start: u64, length: u64) -> SysResult<()> {
+    pub fn reserve_range(
+        &mut self,
+        space: IoSpace,
+        start: u64,
+        length: u64,
+        async_write: bool,
+    ) -> SysResult<()> {
         if length == 0 {
             return Err(SysError::new(EINVAL));
         }
@@ -189,10 +196,16 @@ impl SharedVcpuState {
             IoSpace::Mmio => &mut self.mmio_regions,
         };
 
-        match space.range(..Range(last_address, 0)).next_back().cloned() {
-            Some(Range(existing_start, _)) if existing_start >= start => Err(SysError::new(EPERM)),
+        match space
+            .range(..Range(last_address, 0, false))
+            .next_back()
+            .cloned()
+        {
+            Some(Range(existing_start, _, _)) if existing_start >= start => {
+                Err(SysError::new(EPERM))
+            }
             _ => {
-                space.insert(Range(start, length));
+                space.insert(Range(start, length, async_write));
                 Ok(())
             }
         }
@@ -200,7 +213,7 @@ impl SharedVcpuState {
 
     //// Releases a reservation previously made at `start` in the given `space`.
     pub fn unreserve_range(&mut self, space: IoSpace, start: u64) -> SysResult<()> {
-        let range = Range(start, 0);
+        let range = Range(start, 0, false);
         let space = match space {
             IoSpace::Ioport => &mut self.ioport_regions,
             IoSpace::Mmio => &mut self.mmio_regions,
@@ -233,7 +246,7 @@ impl SharedVcpuState {
     }
 
     fn is_reserved(&self, space: IoSpace, addr: u64) -> bool {
-        if let Some(Range(start, len)) = self.first_before(space, addr) {
+        if let Some(Range(start, len, _)) = self.first_before(space, addr) {
             let offset = addr - start;
             if offset < len {
                 return true;
@@ -249,7 +262,10 @@ impl SharedVcpuState {
         };
 
         match addr.checked_add(1) {
-            Some(next_addr) => space.range(..Range(next_addr, 0)).next_back().cloned(),
+            Some(next_addr) => space
+                .range(..Range(next_addr, 0, false))
+                .next_back()
+                .cloned(),
             None => None,
         }
     }
@@ -403,11 +419,14 @@ impl PluginVcpu {
         let first_before_addr = vcpu_state_lock.first_before(io_space, addr);
 
         match first_before_addr {
-            Some(Range(start, len)) => {
+            Some(Range(start, len, async_write)) => {
                 let offset = addr - start;
                 if offset >= len {
                     return false;
                 }
+                if async_write && !data.is_write() {
+                    return false;
+                }
 
                 let mut wait_reason = VcpuResponse_Wait::new();
                 let io = wait_reason.mut_io();
@@ -418,7 +437,8 @@ impl PluginVcpu {
                 io.address = addr;
                 io.is_write = data.is_write();
                 io.data = data.as_slice().to_vec();
-                if vcpu_state_lock.matches_hint(io_space, addr, io.is_write) {
+                io.no_resume = async_write;
+                if !async_write && vcpu_state_lock.matches_hint(io_space, addr, io.is_write) {
                     if let Ok(regs) = vcpu.get_regs() {
                         let (has_sregs, has_debugregs) = vcpu_state_lock.check_hint_details(&regs);
                         io.regs = VcpuRegs(regs).as_slice().to_vec();
@@ -438,11 +458,34 @@ impl PluginVcpu {
                 // don't hold lock while blocked in `handle_until_resume`.
                 drop(vcpu_state_lock);
 
-                self.wait_reason.set(Some(wait_reason));
-                match self.handle_until_resume(vcpu) {
-                    Ok(resume_data) => data.copy_from_slice(&resume_data),
-                    Err(e) if e.errno() == EPIPE => {}
-                    Err(e) => error!("failed to process vcpu requests: {}", e),
+                if async_write {
+                    let mut response = VcpuResponse::new();
+                    response.set_wait(wait_reason);
+
+                    let mut response_buffer = self.response_buffer.borrow_mut();
+                    response_buffer.clear();
+                    let mut stream = CodedOutputStream::vec(&mut response_buffer);
+                    match response.write_length_delimited_to(&mut stream) {
+                        Ok(_) => {
+                            match stream.flush() {
+                                Ok(_) => {}
+                                Err(e) => error!("failed to flush to vec: {}", e),
+                            }
+                            let mut write_pipe = &self.write_pipe;
+                            match write_pipe.write(&response_buffer[..]) {
+                                Ok(_) => {}
+                                Err(e) => error!("failed to write to pipe: {}", e),
+                            }
+                        }
+                        Err(e) => error!("failed to write to buffer: {}", e),
+                    }
+                } else {
+                    self.wait_reason.set(Some(wait_reason));
+                    match self.handle_until_resume(vcpu) {
+                        Ok(resume_data) => data.copy_from_slice(&resume_data),
+                        Err(e) if e.errno() == EPIPE => {}
+                        Err(e) => error!("failed to process vcpu requests: {}", e),
+                    }
                 }
                 true
             }
@@ -637,9 +680,11 @@ impl PluginVcpu {
         if send_response {
             let mut response_buffer = self.response_buffer.borrow_mut();
             response_buffer.clear();
+            let mut stream = CodedOutputStream::vec(&mut response_buffer);
             response
-                .write_to_vec(&mut response_buffer)
+                .write_length_delimited_to(&mut stream)
                 .map_err(proto_to_sys_err)?;
+            stream.flush().map_err(proto_to_sys_err)?;
             let mut write_pipe = &self.write_pipe;
             write_pipe
                 .write(&response_buffer[..])
@@ -666,37 +711,37 @@ mod tests {
     fn shared_vcpu_reserve() {
         let mut shared_vcpu_state = SharedVcpuState::default();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x10, 0)
+            .reserve_range(IoSpace::Ioport, 0x10, 0, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x10, 0x10)
+            .reserve_range(IoSpace::Ioport, 0x10, 0x10, false)
             .unwrap();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x0f, 0x10)
+            .reserve_range(IoSpace::Ioport, 0x0f, 0x10, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x10, 0x10)
+            .reserve_range(IoSpace::Ioport, 0x10, 0x10, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x10, 0x15)
+            .reserve_range(IoSpace::Ioport, 0x10, 0x15, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x12, 0x15)
+            .reserve_range(IoSpace::Ioport, 0x12, 0x15, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x12, 0x01)
+            .reserve_range(IoSpace::Ioport, 0x12, 0x01, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x0, 0x20)
+            .reserve_range(IoSpace::Ioport, 0x0, 0x20, false)
             .unwrap_err();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x20, 0x05)
+            .reserve_range(IoSpace::Ioport, 0x20, 0x05, false)
             .unwrap();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x25, 0x05)
+            .reserve_range(IoSpace::Ioport, 0x25, 0x05, false)
             .unwrap();
         shared_vcpu_state
-            .reserve_range(IoSpace::Ioport, 0x0, 0x10)
+            .reserve_range(IoSpace::Ioport, 0x0, 0x10, false)
             .unwrap();
     }
 }
diff --git a/tests/plugin_async_write.c b/tests/plugin_async_write.c
new file mode 100644
index 0000000..5c99f25
--- /dev/null
+++ b/tests/plugin_async_write.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright 2019 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <linux/memfd.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "crosvm.h"
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#define F_LINUX_SPECIFIC_BASE 1024
+#endif
+
+#ifndef F_ADD_SEALS
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#endif
+
+#ifndef F_SEAL_SHRINK
+#define F_SEAL_SHRINK 0x0002
+#endif
+
+#define KILL_ADDRESS   0x3f9
+#define ASYNC_ADDRESS  0x500
+
+int g_kill_evt;
+int got_error = 0;
+
+void *vcpu_thread(void *arg) {
+    struct crosvm_vcpu *vcpu = arg;
+    struct crosvm_vcpu_event evt;
+    while (crosvm_vcpu_wait(vcpu, &evt) == 0) {
+        if (evt.kind == CROSVM_VCPU_EVENT_KIND_INIT) {
+            struct kvm_sregs sregs;
+            crosvm_vcpu_get_sregs(vcpu, &sregs);
+            sregs.cs.base = 0;
+            sregs.cs.selector = 0;
+            sregs.es.base = KILL_ADDRESS;
+            sregs.es.selector = 0;
+            crosvm_vcpu_set_sregs(vcpu, &sregs);
+
+            struct kvm_regs regs;
+            crosvm_vcpu_get_regs(vcpu, &regs);
+            regs.rip = 0x1000;
+            regs.rax = 2;
+            regs.rbx = 7;
+            regs.rflags = 2;
+            crosvm_vcpu_set_regs(vcpu, &regs);
+        }
+        if (evt.kind == CROSVM_VCPU_EVENT_KIND_IO_ACCESS) {
+            if (evt.io_access.address_space == CROSVM_ADDRESS_SPACE_IOPORT &&
+                evt.io_access.address == ASYNC_ADDRESS &&
+                evt.io_access.is_write &&
+                evt.io_access.length == 1) {
+                int ret;
+                if (!evt.io_access.no_resume) {
+                    fprintf(stderr, "should have been told not to resume\n");
+                    got_error = 1;
+                }
+
+                ret = crosvm_vcpu_wait(vcpu, &evt);
+                if (ret == 0) {
+                    if (evt.kind != CROSVM_VCPU_EVENT_KIND_IO_ACCESS ||
+                        evt.io_access.address_space !=
+                        CROSVM_ADDRESS_SPACE_IOPORT ||
+                        evt.io_access.address != ASYNC_ADDRESS ||
+                        !evt.io_access.is_write ||
+                        !evt.io_access.no_resume ||
+                        evt.io_access.length != 1) {
+                        fprintf(stderr, "got unexpected wait #1 result\n");
+                        got_error = 1;
+                    }
+                } else {
+                    fprintf(stderr, "crosvm_vcpu_wait() #1 failed: %d\n", ret);
+                    got_error = 1;
+                }
+
+                ret = crosvm_vcpu_wait(vcpu, &evt);
+                if (ret == 0) {
+                    if (evt.kind != CROSVM_VCPU_EVENT_KIND_IO_ACCESS ||
+                        evt.io_access.address_space !=
+                        CROSVM_ADDRESS_SPACE_IOPORT ||
+                        evt.io_access.address != ASYNC_ADDRESS ||
+                        !evt.io_access.is_write ||
+                        !evt.io_access.no_resume ||
+                        evt.io_access.length != 1) {
+                        fprintf(stderr, "got unexpected wait #2 result\n");
+                        got_error = 1;
+                    }
+                } else {
+                    fprintf(stderr, "crosvm_vcpu_wait() #2 failed: %d\n", ret);
+                    got_error = 1;
+                }
+
+                // skip the crosvm_vcpu_resume()
+                continue;
+            }
+            if (evt.io_access.address_space == CROSVM_ADDRESS_SPACE_IOPORT &&
+                evt.io_access.address == KILL_ADDRESS &&
+                evt.io_access.is_write &&
+                evt.io_access.length == 1 &&
+                evt.io_access.data[0] == 1)
+            {
+                uint64_t dummy = 1;
+                write(g_kill_evt, &dummy, sizeof(dummy));
+                return NULL;
+            }
+        }
+
+        crosvm_vcpu_resume(vcpu);
+    }
+
+    return NULL;
+}
+
+int main(int argc, char** argv) {
+    const uint8_t code[] = {
+    /*
+    B007    mov al,0x7
+    BA0005  mov dx,0x500
+    EE      out dx,al
+    EE      out dx,al
+    EE      out dx,al
+    BAF903  mov dx,0x3f9
+    B001    mov al,0x1
+    EE      out dx,al
+    F4      hlt
+    */
+        0xb0, 0x7,
+        0xba, (ASYNC_ADDRESS & 0xFF), ((ASYNC_ADDRESS >> 8) & 0xFF),
+        0xee,
+        0xee,
+        0xee,
+        0xba, (KILL_ADDRESS & 0xFF), ((KILL_ADDRESS >> 8) & 0xFF),
+        0xb0, 0x01,
+        0xee,
+        0xf4
+    };
+
+    struct crosvm *crosvm;
+    int ret = crosvm_connect(&crosvm);
+    if (ret) {
+        fprintf(stderr, "failed to connect to crosvm: %d\n", ret);
+        return 1;
+    }
+
+    /*
+     * Not strictly necessary, but demonstrates we can have as many connections
+     * as we please.
+     */
+    struct crosvm *extra_crosvm;
+    ret = crosvm_new_connection(crosvm, &extra_crosvm);
+    if (ret) {
+        fprintf(stderr, "failed to make new socket: %d\n", ret);
+        return 1;
+    }
+
+    /* We needs this eventfd to know when to exit before being killed. */
+    g_kill_evt = crosvm_get_shutdown_eventfd(crosvm);
+    if (g_kill_evt < 0) {
+        fprintf(stderr, "failed to get kill eventfd: %d\n", g_kill_evt);
+        return 1;
+    }
+
+    ret = crosvm_reserve_async_write_range(crosvm, CROSVM_ADDRESS_SPACE_IOPORT,
+                                           ASYNC_ADDRESS, 1);
+    if (ret) {
+        fprintf(stderr, "failed to reserve async ioport range: %d\n", ret);
+        return 1;
+    }
+
+    ret = crosvm_reserve_range(crosvm, CROSVM_ADDRESS_SPACE_IOPORT,
+                               KILL_ADDRESS, 1);
+    if (ret) {
+        fprintf(stderr, "failed to reserve kill ioport range: %d\n", ret);
+        return 1;
+    }
+
+    int mem_size = 0x2000;
+    int mem_fd = syscall(SYS_memfd_create, "guest_mem",
+                         MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (mem_fd < 0) {
+        fprintf(stderr, "failed to create guest memfd: %d\n", errno);
+        return 1;
+    }
+    ret = ftruncate(mem_fd, mem_size);
+    if (ret) {
+        fprintf(stderr, "failed to set size of guest memory: %d\n", errno);
+        return 1;
+    }
+    uint8_t *mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                        mem_fd, 0x1000);
+    if (mem == MAP_FAILED) {
+        fprintf(stderr, "failed to mmap guest memory: %d\n", errno);
+        return 1;
+    }
+    fcntl(mem_fd, F_ADD_SEALS, F_SEAL_SHRINK);
+    memcpy(mem, code, sizeof(code));
+
+    struct crosvm_memory *mem_obj;
+    ret = crosvm_create_memory(crosvm, mem_fd, 0x1000, 0x1000, 0x1000, false,
+                               false, &mem_obj);
+    if (ret) {
+        fprintf(stderr, "failed to create memory in crosvm: %d\n", ret);
+        return 1;
+    }
+
+    /* get and creat a thread for each vcpu */
+    struct crosvm_vcpu *vcpus[32];
+    pthread_t vcpu_threads[32];
+    uint32_t vcpu_count;
+    for (vcpu_count = 0; vcpu_count < 32; vcpu_count++) {
+        ret = crosvm_get_vcpu(crosvm, vcpu_count, &vcpus[vcpu_count]);
+        if (ret == -ENOENT)
+            break;
+
+        if (ret) {
+            fprintf(stderr, "error while getting all vcpus: %d\n", ret);
+            return 1;
+        }
+        pthread_create(&vcpu_threads[vcpu_count], NULL, vcpu_thread,
+                       vcpus[vcpu_count]);
+    }
+
+    ret = crosvm_start(extra_crosvm);
+    if (ret) {
+        fprintf(stderr, "failed to tell crosvm to start: %d\n", ret);
+        return 1;
+    }
+
+    /* Wait for crosvm to request that we exit otherwise we will be killed. */
+    uint64_t dummy;
+    read(g_kill_evt, &dummy, 8);
+
+    ret = crosvm_destroy_memory(crosvm, &mem_obj);
+    if (ret) {
+        fprintf(stderr, "failed to destroy memory in crosvm: %d\n", ret);
+        return 1;
+    }
+
+    ret = crosvm_reserve_async_write_range(crosvm, CROSVM_ADDRESS_SPACE_IOPORT,
+                                           ASYNC_ADDRESS, 0);
+    if (ret) {
+        fprintf(stderr, "failed to unreserve async ioport range: %d\n", ret);
+        return 1;
+    }
+
+    ret = crosvm_reserve_range(crosvm, CROSVM_ADDRESS_SPACE_IOPORT,
+                               KILL_ADDRESS, 0);
+    if (ret) {
+        fprintf(stderr, "failed to unreserve kill ioport range: %d\n", ret);
+        return 1;
+    }
+
+    if (got_error) {
+      fprintf(stderr, "vm ran to completion but with an error\n");
+      return 1;
+    }
+
+    return 0;
+}
diff --git a/tests/plugins.rs b/tests/plugins.rs
index 47b0ca3..d56f4ce 100644
--- a/tests/plugins.rs
+++ b/tests/plugins.rs
@@ -231,6 +231,11 @@ fn test_hint() {
 }
 
 #[test]
+fn test_async_write() {
+    test_plugin(include_str!("plugin_async_write.c"));
+}
+
+#[test]
 fn test_dirty_log() {
     test_plugin(include_str!("plugin_dirty_log.c"));
 }