aarch64: add PCI device memory region to fdt

The device memory SystemAllocator allocates device addresses past the
end of physical memory. If a device tries to actually use this address
to register a new PCI BAR, the kernel will fail with a "can't claim BAR,
no compatible bridge window" error. On ARM, the kernel must be informed
of where PCI bars are allowed to be allocated through the Device Tree.

This CL adds a new PCI memory region to the fdt for device memory.

BUG=chromium:936567
TEST=see CL:1493014. Run on ARM.

Change-Id: I0faa6f7082ae53f7a792cec53a21adba109bc00d
Reviewed-on: https://chromium-review.googlesource.com/1558940
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>

2 files changed, 33 insertions, 13 deletions

diff --git a/aarch64/src/fdt.rs b/aarch64/src/fdt.rs
index 3b0126f..a41494d 100644
--- a/aarch64/src/fdt.rs
+++ b/aarch64/src/fdt.rs
@@ -185,21 +185,32 @@ fn create_chosen_node(
     Ok(())
 }
 
-fn create_pci_nodes(fdt: &mut Vec<u8>, pci_irqs: Vec<(u32, PciInterruptPin)>) -> Result<()> {
+fn create_pci_nodes(
+    fdt: &mut Vec<u8>,
+    pci_irqs: Vec<(u32, PciInterruptPin)>,
+    pci_device_base: u64,
+    pci_device_size: u64,
+) -> Result<()> {
     // Add devicetree nodes describing a PCI generic host controller.
     // See Documentation/devicetree/bindings/pci/host-generic-pci.txt in the kernel
     // and "PCI Bus Binding to IEEE Std 1275-1994".
     let ranges = generate_prop32(&[
-        // bus address (ss = 01: 32-bit memory space)
-        0x2000000,
-        (AARCH64_MMIO_BASE >> 32) as u32,
+        // mmio addresses
+        0x2000000,                        // (ss = 01: 32-bit memory space)
+        (AARCH64_MMIO_BASE >> 32) as u32, // PCI address
         AARCH64_MMIO_BASE as u32,
-        // CPU address
-        (AARCH64_MMIO_BASE >> 32) as u32,
+        (AARCH64_MMIO_BASE >> 32) as u32, // CPU address
         AARCH64_MMIO_BASE as u32,
-        // size
-        (AARCH64_MMIO_SIZE >> 32) as u32,
+        (AARCH64_MMIO_SIZE >> 32) as u32, // size
         AARCH64_MMIO_SIZE as u32,
+        // device addresses
+        0x3000000,                      // (ss = 11: 64-bit memory space)
+        (pci_device_base >> 32) as u32, // PCI address
+        pci_device_base as u32,
+        (pci_device_base >> 32) as u32, // CPU address
+        pci_device_base as u32,
+        (pci_device_size >> 32) as u32, // size
+        pci_device_size as u32,
     ]);
     let bus_range = generate_prop32(&[0, 0]); // Only bus 0
     let reg = generate_prop64(&[AARCH64_PCI_CFG_BASE, AARCH64_PCI_CFG_SIZE]);
@@ -293,6 +304,8 @@ fn create_rtc_node(fdt: &mut Vec<u8>) -> Result<()> {
 /// * `pci_irqs` - List of PCI device number to PCI interrupt pin mappings
 /// * `num_cpus` - Number of virtual CPUs the guest will have
 /// * `fdt_load_offset` - The offset into physical memory for the device tree
+/// * `pci_device_base` - The offset into physical memory for PCI device memory
+/// * `pci_device_size` - The size of PCI device memory
 /// * `cmdline` - The kernel commandline
 /// * `initrd` - An optional tuple of initrd guest physical address and size
 pub fn create_fdt(
@@ -301,6 +314,8 @@ pub fn create_fdt(
     pci_irqs: Vec<(u32, PciInterruptPin)>,
     num_cpus: u32,
     fdt_load_offset: u64,
+    pci_device_base: u64,
+    pci_device_size: u64,
     cmdline: &CStr,
     initrd: Option<(GuestAddress, usize)>,
 ) -> Result<()> {
@@ -321,7 +336,7 @@ pub fn create_fdt(
     create_timer_node(&mut fdt, num_cpus)?;
     create_serial_node(&mut fdt)?;
     create_psci_node(&mut fdt)?;
-    create_pci_nodes(&mut fdt, pci_irqs)?;
+    create_pci_nodes(&mut fdt, pci_irqs, pci_device_base, pci_device_size)?;
     create_rtc_node(&mut fdt)?;
     // End giant node
     end_node(&mut fdt)?;
diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs
index 39deca0..ac70e0d 100644
--- a/aarch64/src/lib.rs
+++ b/aarch64/src/lib.rs
@@ -311,12 +311,15 @@ impl AArch64 {
             }
             None => None,
         };
+        let (pci_device_base, pci_device_size) = Self::get_device_addr_base_size(mem_size);
         fdt::create_fdt(
             AARCH64_FDT_MAX_SIZE as usize,
             mem,
             pci_irqs,
             vcpu_count,
             fdt_offset(mem_size),
+            pci_device_base,
+            pci_device_size,
             cmdline,
             initrd,
         )
@@ -330,8 +333,10 @@ impl AArch64 {
         Ok(mem)
     }
 
-    fn get_base_dev_pfn(mem_size: u64) -> u64 {
-        (AARCH64_PHYS_MEM_START + mem_size) >> 12
+    fn get_device_addr_base_size(mem_size: u64) -> (u64, u64) {
+        let base = AARCH64_PHYS_MEM_START + mem_size;
+        let size = u64::max_value() - base;
+        (base, size)
     }
 
     /// This returns a base part of the kernel command for this architecture
@@ -343,9 +348,9 @@ impl AArch64 {
 
     /// Returns a system resource allocator.
     fn get_resource_allocator(mem_size: u64, gpu_allocation: bool) -> SystemAllocator {
-        let device_addr_start = Self::get_base_dev_pfn(mem_size) * sys_util::pagesize() as u64;
+        let (device_addr_base, device_addr_size) = Self::get_device_addr_base_size(mem_size);
         SystemAllocator::builder()
-            .add_device_addresses(device_addr_start, u64::max_value() - device_addr_start)
+            .add_device_addresses(device_addr_base, device_addr_size)
             .add_mmio_addresses(AARCH64_MMIO_BASE, AARCH64_MMIO_SIZE)
             .create_allocator(AARCH64_IRQ_BASE, gpu_allocation)
             .unwrap()