cloud-hypervisor · rbradford · Jan 14, 2025 · Jan 8, 2025 · Jan 8, 2025 · Jan 8, 2025
@@ -66,6 +66,7 @@ fuzz_target!(|bytes: &[u8]| -> Corpus {
         SeccompAction::Allow,
         EventFd::new(EFD_NONBLOCK).unwrap(),
         ((MEM_SIZE - IOVA_SPACE_SIZE) as u64, (MEM_SIZE - 1) as u64),
+        64,
         None,
     )
     .unwrap();

@@ -204,7 +204,7 @@ fn create_app(default_vcpus: String, default_memory: String, default_rng: String
         .arg(
             Arg::new("platform")
                 .long("platform")
-                .help("num_pci_segments=<num_pci_segments>,iommu_segments=<list_of_segments>,serial_number=<dmi_device_serial_number>,uuid=<dmi_device_uuid>,oem_strings=<list_of_strings>")
+                .help("num_pci_segments=<num_pci_segments>,iommu_segments=<list_of_segments>,iommu_address_width=<bits>,serial_number=<dmi_device_serial_number>,uuid=<dmi_device_uuid>,oem_strings=<list_of_strings>")
                 .num_args(1)
                 .group("vm-config"),
         )

@@ -9111,7 +9111,10 @@ mod vfio {
             .args(["--cpus", "boot=4"])
             .args(["--memory", "size=4G"])
             .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
-            .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()])
+            .args([
+                "--device",
+                format!("path={NVIDIA_VFIO_DEVICE},iommu=on").as_str(),
+            ])
             .args(["--api-socket", &api_socket])
             .default_disks()
             .default_net()
@@ -9136,6 +9139,43 @@ mod vfio {
 
         handle_child_output(r, &output);
     }
+
+    #[test]
+    fn test_nvidia_card_iommu_address_width() {
+        let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(jammy));
+        let api_socket = temp_api_path(&guest.tmp_dir);
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--cpus", "boot=4"])
+            .args(["--memory", "size=4G"])
+            .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()])
+            .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()])
+            .args([
+                "--platform",
+                "num_pci_segments=2,iommu_segments=1,iommu_address_width=42",
+            ])
+            .args(["--api-socket", &api_socket])
+            .default_disks()
+            .default_net()
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+
+            assert!(guest
+                .ssh_command("sudo dmesg")
+                .unwrap()
+                .contains("input address: 42 bits"));
+        });
+
+        let _ = child.kill();
+        let output = child.wait_with_output().unwrap();
+
+        handle_child_output(r, &output);
+    }
 }
 
 mod live_migration {

@@ -906,9 +906,10 @@ impl Iommu {
         seccomp_action: SeccompAction,
         exit_evt: EventFd,
         msi_iova_space: (u64, u64),
+        address_width_bits: u8,
         state: Option<IommuState>,
     ) -> io::Result<(Self, Arc<IommuMapping>)> {
-        let (avail_features, acked_features, endpoints, domains, paused) =
+        let (mut avail_features, acked_features, endpoints, domains, paused) =
             if let Some(state) = state {
                 info!("Restoring virtio-iommu {}", id);
                 (
@@ -939,12 +940,20 @@ impl Iommu {
                 (avail_features, 0, BTreeMap::new(), BTreeMap::new(), false)
             };
 
-        let config = VirtioIommuConfig {
+        let mut config = VirtioIommuConfig {
             page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
             probe_size: PROBE_PROP_SIZE,
             ..Default::default()
         };
 
+        if address_width_bits < 64 {
+            avail_features |= 1u64 << VIRTIO_IOMMU_F_INPUT_RANGE;
+            config.input_range = VirtioIommuRange64 {
+                start: 0,
+                end: (1u64 << address_width_bits) - 1,
+            }
+        }
+
         let mapping = Arc::new(IommuMapping {
             endpoints: Arc::new(RwLock::new(endpoints)),
             domains: Arc::new(RwLock::new(domains)),

@@ -718,6 +718,9 @@ components:
           items:
             type: integer
             format: int16
+        iommu_address_width:
+          type: integer
+          format: uint8
         serial_number:
           type: string
         uuid:

@@ -21,6 +21,7 @@ use crate::landlock::LandlockAccess;
 use crate::vm_config::*;
 
 const MAX_NUM_PCI_SEGMENTS: u16 = 96;
+const MAX_IOMMU_ADDRESS_WIDTH_BITS: u8 = 64;
 
 /// Errors associated with VM configuration parameters.
 #[derive(Debug, Error)]
@@ -183,6 +184,8 @@ pub enum ValidationError {
     InvalidPciSegment(u16),
     /// Invalid PCI segment aperture weight
     InvalidPciSegmentApertureWeight(u32),
+    /// Invalid IOMMU address width in bits
+    InvalidIommuAddressWidthBits(u8),
     /// Balloon too big
     BalloonLargerThanRam(u64, u64),
     /// On a IOMMU segment but not behind IOMMU
@@ -309,6 +312,9 @@ impl fmt::Display for ValidationError {
             InvalidPciSegmentApertureWeight(aperture_weight) => {
                 write!(f, "Invalid PCI segment aperture weight: {aperture_weight}")
             }
+            InvalidIommuAddressWidthBits(iommu_address_width_bits) => {
+                write!(f, "IOMMU address width in bits ({iommu_address_width_bits}) should be less than or equal to {MAX_IOMMU_ADDRESS_WIDTH_BITS}")
+            }
             BalloonLargerThanRam(balloon_size, ram_size) => {
                 write!(
                     f,
@@ -817,6 +823,7 @@ impl PlatformConfig {
         parser
             .add("num_pci_segments")
             .add("iommu_segments")
+            .add("iommu_address_width")
             .add("serial_number")
             .add("uuid")
             .add("oem_strings");
@@ -834,6 +841,10 @@ impl PlatformConfig {
             .convert::<IntegerList>("iommu_segments")
             .map_err(Error::ParsePlatform)?
             .map(|v| v.0.iter().map(|e| *e as u16).collect());
+        let iommu_address_width_bits: u8 = parser
+            .convert("iommu_address_width")
+            .map_err(Error::ParsePlatform)?
+            .unwrap_or(MAX_IOMMU_ADDRESS_WIDTH_BITS);
         let serial_number = parser
             .convert("serial_number")
             .map_err(Error::ParsePlatform)?;
@@ -857,6 +868,7 @@ impl PlatformConfig {
         Ok(PlatformConfig {
             num_pci_segments,
             iommu_segments,
+            iommu_address_width_bits,
             serial_number,
             uuid,
             oem_strings,
@@ -882,6 +894,12 @@ impl PlatformConfig {
             }
         }
 
+        if self.iommu_address_width_bits > MAX_IOMMU_ADDRESS_WIDTH_BITS {
+            return Err(ValidationError::InvalidIommuAddressWidthBits(
+                self.iommu_address_width_bits,
+            ));
+        }
+
         Ok(())
     }
 }
@@ -3998,6 +4016,7 @@ mod tests {
         PlatformConfig {
             num_pci_segments: MAX_NUM_PCI_SEGMENTS,
             iommu_segments: None,
+            iommu_address_width_bits: MAX_IOMMU_ADDRESS_WIDTH_BITS,
             serial_number: None,
             uuid: None,
             oem_strings: None,
@@ -4296,6 +4315,18 @@ mod tests {
             Err(ValidationError::InvalidPciSegment(MAX_NUM_PCI_SEGMENTS + 1))
         );
 
+        let mut invalid_config = valid_config.clone();
+        invalid_config.platform = Some(PlatformConfig {
+            iommu_address_width_bits: MAX_IOMMU_ADDRESS_WIDTH_BITS + 1,
+            ..platform_fixture()
+        });
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::InvalidIommuAddressWidthBits(
+                MAX_IOMMU_ADDRESS_WIDTH_BITS + 1
+            ))
+        );
+
         let mut still_valid_config = valid_config.clone();
         still_valid_config.platform = Some(PlatformConfig {
             iommu_segments: Some(vec![1, 2, 3]),

@@ -100,7 +100,8 @@ use crate::pci_segment::PciSegment;
 use crate::serial_manager::{Error as SerialManagerError, SerialManager};
 use crate::vm_config::{
     ConsoleOutputMode, DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, UserDeviceConfig,
-    VdpaConfig, VhostMode, VmConfig, VsockConfig, DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT,
+    VdpaConfig, VhostMode, VmConfig, VsockConfig, DEFAULT_IOMMU_ADDRESS_WIDTH_BITS,
+    DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT,
 };
 use crate::{device_node, GuestRegionMmap, PciDeviceInfo, DEVICE_MANAGER_SNAPSHOT_ID};
 
@@ -1365,6 +1366,13 @@ impl DeviceManager {
     ) -> DeviceManagerResult<()> {
         let iommu_id = String::from(IOMMU_DEVICE_NAME);
 
+        let iommu_address_width_bits =
+            if let Some(ref platform) = self.config.lock().unwrap().platform {
+                platform.iommu_address_width_bits
+            } else {
+                DEFAULT_IOMMU_ADDRESS_WIDTH_BITS
+            };
+
         let iommu_device = if self.config.lock().unwrap().iommu {
             let (device, mapping) = virtio_devices::Iommu::new(
                 iommu_id.clone(),
@@ -1373,6 +1381,7 @@ impl DeviceManager {
                     .try_clone()
                     .map_err(DeviceManagerError::EventFd)?,
                 self.get_msi_iova_space(),
+                iommu_address_width_bits,
                 state_from_id(self.snapshot.as_ref(), iommu_id.as_str())
                     .map_err(DeviceManagerError::RestoreGetState)?,
             )

@@ -88,12 +88,19 @@ pub fn default_platformconfig_num_pci_segments() -> u16 {
     DEFAULT_NUM_PCI_SEGMENTS
 }
 
+pub const DEFAULT_IOMMU_ADDRESS_WIDTH_BITS: u8 = 64;
+pub fn default_platformconfig_iommu_address_width_bits() -> u8 {
+    DEFAULT_IOMMU_ADDRESS_WIDTH_BITS
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct PlatformConfig {
     #[serde(default = "default_platformconfig_num_pci_segments")]
     pub num_pci_segments: u16,
     #[serde(default)]
     pub iommu_segments: Option<Vec<u16>>,
+    #[serde(default = "default_platformconfig_iommu_address_width_bits")]
+    pub iommu_address_width_bits: u8,
     #[serde(default)]
     pub serial_number: Option<String>,
     #[serde(default)]