cloud-hypervisor · rbradford · Dec 3, 2025 · Oct 7, 2025 · Sep 24, 2025 · Sep 24, 2025
@@ -113,8 +113,8 @@ acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"
 kvm-bindings = "0.12.1"
 kvm-ioctls = "0.22.1"
 linux-loader = "0.13.1"
-mshv-bindings = "0.6.0"
-mshv-ioctls = "0.6.0"
+mshv-bindings = "0.6.5"
+mshv-ioctls = "0.6.5"
 seccompiler = "0.5.0"
 vfio-bindings = { version = "0.6.0", default-features = false }
 vfio-ioctls = { version = "0.5.1", default-features = false }

@@ -44,6 +44,8 @@ pub const MAX_SUPPORTED_CPUS_LEGACY: u32 = 254;
 #[cfg(feature = "kvm")]
 const TSC_DEADLINE_TIMER_ECX_BIT: u8 = 24; // tsc deadline timer ecx bit.
 const HYPERVISOR_ECX_BIT: u8 = 31; // Hypervisor ecx bit.
+const VMX_ECX_BIT: u8 = 5; // VMX for Intel
+const SVM_ECX_BIT: u8 = 2; // SVM for AMD
 const MTRR_EDX_BIT: u8 = 12; // Hypervisor ecx bit.
 const INVARIANT_TSC_EDX_BIT: u8 = 8; // Invariant TSC bit on 0x8000_0007 EDX
 const AMX_BF16: u8 = 22; // AMX tile computation on bfloat16 numbers
@@ -806,6 +808,7 @@ pub fn generate_common_cpuid(
     Ok(cpuid)
 }
 
+#[allow(clippy::too_many_arguments)]
 pub fn configure_vcpu(
     vcpu: &dyn hypervisor::Vcpu,
     id: u32,
@@ -814,6 +817,7 @@ pub fn configure_vcpu(
     kvm_hyperv: bool,
     cpu_vendor: CpuVendor,
     topology: (u16, u16, u16, u16),
+    nested: bool,
 ) -> super::Result<()> {
     let x2apic_id = get_x2apic_id(id, Some(topology));
 
@@ -832,6 +836,17 @@ pub fn configure_vcpu(
             entry.ebx &= 0xffffff;
             entry.ebx |= x2apic_id << 24;
             apic_id_patched = true;
+            if !nested {
+                // Disable nested virtualization for Intel
+                entry.ecx &= !(1 << VMX_ECX_BIT);
+            }
+            break;
+        }
+        if entry.function == 0x8000_0001 {
+            if !nested {
+                // Disable the nested virtualization for AMD
+                entry.ecx &= !(1 << SVM_ECX_BIT);
+            }
             break;
         }
     }

@@ -18,11 +18,12 @@ struct CpusConfig {
     max_phys_bits: u8,
     affinity: Option<Vec<CpuAffinity>>,
     features: CpuFeatures,
+    nested: bool,
 }
 ```
 
 ```
---cpus boot=<boot_vcpus>,max=<max_vcpus>,topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>,kvm_hyperv=on|off,max_phys_bits=<maximum_number_of_physical_bits>,affinity=<list_of_vcpus_with_their_associated_cpuset>,features=<list_of_features_to_enable>
+--cpus boot=<boot_vcpus>,max=<max_vcpus>,topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>,kvm_hyperv=on|off,max_phys_bits=<maximum_number_of_physical_bits>,affinity=<list_of_vcpus_with_their_associated_cpuset>,features=<list_of_features_to_enable>,nested=on|off
 ```
 
 ### `boot`
@@ -209,3 +210,14 @@ _Example_
 ```
 
 In this example the amx CPU feature will be enabled for the VMM.
+
+
+### `nested`
+
+Enable nested virtualization (default on). Nested virtualization is needed to access hardware virtualization by this guest. This option can only be changed on x86-64.
+
+_Example_
+
+```
+--cpus nested=on
+```
@@ -25,7 +25,7 @@ libc = "0.2.177"
 libfuzzer-sys = "0.4.10"
 linux-loader = { version = "0.13.1", features = ["bzimage", "elf", "pe"] }
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
-mshv-bindings = "0.6.0"
+mshv-bindings = "0.6.5"
 net_util = { path = "../net_util" }
 seccompiler = "0.5.0"
 virtio-devices = { path = "../virtio-devices" }

@@ -132,6 +132,7 @@ impl RequestHandler for StubApiRequestHandler {
                     max_phys_bits: 46,
                     affinity: None,
                     features: CpuFeatures::default(),
+                    nested: true,
                 },
                 memory: MemoryConfig {
                     size: 536_870_912,

@@ -170,6 +170,7 @@ pub struct HypervisorVmConfig {
     pub sev_snp_enabled: bool,
     #[cfg(feature = "sev_snp")]
     pub mem_size: u64,
+    pub nested: bool,
 }
 
 #[derive(Copy, Clone)]

@@ -14,7 +14,10 @@ use log::{debug, warn};
 use mshv_bindings::*;
 #[cfg(target_arch = "x86_64")]
 use mshv_ioctls::InterruptRequest;
-use mshv_ioctls::{Mshv, NoDatamatch, VcpuFd, VmFd, VmType, set_registers_64};
+use mshv_ioctls::{
+    Mshv, NoDatamatch, VcpuFd, VmFd, VmType, make_default_partition_create_arg,
+    make_default_synthetic_features_mask, set_registers_64,
+};
 use vfio_ioctls::VfioDeviceFd;
 use vm::DataMatch;
 #[cfg(feature = "sev_snp")]
@@ -292,10 +295,37 @@ impl hypervisor::Hypervisor for MshvHypervisor {
                 VmType::Normal
             };
         }
-
+        let mut create_args = make_default_partition_create_arg(mshv_vm_type);
+        let mut disable_proc_features = hv_partition_processor_features::default();
+        // SAFETY: Accessing a union element from bindgen generated bindings.
+        unsafe {
+            for i in 0..create_args.pt_num_cpu_fbanks {
+                disable_proc_features.as_uint64[i as usize] = create_args.pt_cpu_fbanks[i as usize];
+            }
+            #[cfg(target_arch = "x86_64")]
+            {
+                // Modify create_args based on user configuration
+                // For now we only handle nested virtualization, but more features can be added here
+                if _config.nested {
+                    create_args.pt_flags |= 1 << MSHV_PT_BIT_NESTED_VIRTUALIZATION;
+                    disable_proc_features
+                        .__bindgen_anon_1
+                        .set_nested_virt_support(0u64);
+                } else {
+                    disable_proc_features
+                        .__bindgen_anon_1
+                        .set_nested_virt_support(1u64);
+                }
+            }
+            // Modified feature bit fields are written back to create_args
+            for i in 0..create_args.pt_num_cpu_fbanks {
+                create_args.pt_cpu_fbanks[i as usize] = disable_proc_features.as_uint64[i as usize];
+            }
+        }
+        let synthetic_features_mask = make_default_synthetic_features_mask();
         let fd: VmFd;
         loop {
-            match self.mshv.create_vm_with_type(mshv_vm_type) {
+            match self.mshv.create_vm_with_args(&create_args) {
                 Ok(res) => fd = res,
                 Err(e) => {
                     if e.errno() == libc::EINTR {
@@ -309,7 +339,11 @@ impl hypervisor::Hypervisor for MshvHypervisor {
             }
             break;
         }
-
+        fd.set_partition_property(
+            hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES,
+            synthetic_features_mask,
+        )
+        .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
         let vm_fd = Arc::new(fd);
 
         #[cfg(target_arch = "x86_64")]

@@ -213,7 +213,8 @@ fn get_cli_options_sorted(
                     topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>,\
                     kvm_hyperv=on|off,max_phys_bits=<maximum_number_of_physical_bits>,\
                     affinity=<list_of_vcpus_with_their_associated_cpuset>,\
-                    features=<list_of_features_to_enable>",
+                    features=<list_of_features_to_enable>,\
+                    nested=on|off",
             )
             .default_value(default_vcpus)
             .group("vm-config"),
@@ -962,6 +963,7 @@ mod unit_tests {
                 max_phys_bits: 46,
                 affinity: None,
                 features: CpuFeatures::default(),
+                nested: true,
             },
             memory: MemoryConfig {
                 size: 536_870_912,

@@ -598,7 +598,8 @@ impl CpusConfig {
             .add("kvm_hyperv")
             .add("max_phys_bits")
             .add("affinity")
-            .add("features");
+            .add("features")
+            .add("nested");
         parser.parse(cpus).map_err(Error::ParseCpus)?;
 
         let boot_vcpus: u32 = parser
@@ -653,6 +654,19 @@ impl CpusConfig {
             }?;
         }
 
+        let nested = parser
+            .convert::<Toggle>("nested")
+            .map_err(Error::ParseCpus)?
+            .is_none_or(|toggle| toggle.0);
+
+        // Nested virtualization is always turned on for aarch64 and riscv64
+        // TODO: revisit this when nested support can be turned of on these architectures
+        #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
+        if !nested {
+            return Err(Error::ParseCpus(OptionParserError::InvalidValue(
+                "nested=off is not supported on aarch64 and riscv64 architectures".to_string(),
+            )));
+        }
         Ok(CpusConfig {
             boot_vcpus,
             max_vcpus,
@@ -661,6 +675,7 @@ impl CpusConfig {
             max_phys_bits,
             affinity,
             features,
+            nested,
         })
     }
 }

@@ -455,6 +455,7 @@ impl Vcpu {
         #[cfg(target_arch = "x86_64")] cpuid: Vec<CpuIdEntry>,
         #[cfg(target_arch = "x86_64")] kvm_hyperv: bool,
         #[cfg(target_arch = "x86_64")] topology: (u16, u16, u16, u16),
+        #[cfg(target_arch = "x86_64")] nested: bool,
     ) -> Result<()> {
         #[cfg(target_arch = "aarch64")]
         {
@@ -475,6 +476,7 @@ impl Vcpu {
             kvm_hyperv,
             self.vendor,
             topology,
+            nested,
         )
         .map_err(Error::VcpuConfiguration)?;
 
@@ -995,6 +997,7 @@ impl CpuManager {
             self.cpuid.clone(),
             self.config.kvm_hyperv,
             topology,
+            self.config.nested,
         )?;
 
         #[cfg(target_arch = "aarch64")]

@@ -210,6 +210,21 @@ pub enum Error {
     #[error("Error applying landlock")]
     ApplyLandlock(#[source] LandlockError),
 }
+
+impl From<&VmConfig> for hypervisor::HypervisorVmConfig {
+    fn from(_value: &VmConfig) -> Self {
+        hypervisor::HypervisorVmConfig {
+            #[cfg(feature = "tdx")]
+            tdx_enabled: _value.platform.as_ref().is_some_and(|p| p.tdx),
+            #[cfg(feature = "sev_snp")]
+            sev_snp_enabled: _value.is_sev_snp_enabled(),
+            #[cfg(feature = "sev_snp")]
+            mem_size: _value.memory.total_size(),
+            nested: _value.cpus.nested,
+        }
+    }
+}
+
 pub type Result<T> = result::Result<T, Error>;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -1006,12 +1021,7 @@ impl Vmm {
 
         let vm = Vm::create_hypervisor_vm(
             self.hypervisor.as_ref(),
-            #[cfg(feature = "tdx")]
-            false,
-            #[cfg(feature = "sev_snp")]
-            false,
-            #[cfg(feature = "sev_snp")]
-            config.lock().unwrap().memory.total_size(),
+            (&*self.vm_config.as_ref().unwrap().lock().unwrap()).into(),
         )
         .map_err(|e| {
             MigratableError::MigrateReceive(anyhow!(
@@ -2388,6 +2398,7 @@ mod unit_tests {
                 max_phys_bits: 46,
                 affinity: None,
                 features: CpuFeatures::default(),
+                nested: true,
             },
             memory: MemoryConfig {
                 size: 536_870_912,

@@ -1009,21 +1009,9 @@ impl Vm {
             vm_config.lock().unwrap().is_tdx_enabled()
         };
 
-        #[cfg(feature = "sev_snp")]
-        let sev_snp_enabled = if snapshot.is_some() {
-            false
-        } else {
-            vm_config.lock().unwrap().is_sev_snp_enabled()
-        };
-
         let vm = Self::create_hypervisor_vm(
             hypervisor.as_ref(),
-            #[cfg(feature = "tdx")]
-            tdx_enabled,
-            #[cfg(feature = "sev_snp")]
-            sev_snp_enabled,
-            #[cfg(feature = "sev_snp")]
-            vm_config.lock().unwrap().memory.total_size(),
+            vm_config.as_ref().lock().unwrap().deref().into(),
         )?;
 
         #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
@@ -1083,19 +1071,9 @@ impl Vm {
 
     pub fn create_hypervisor_vm(
         hypervisor: &dyn hypervisor::Hypervisor,
-        #[cfg(feature = "tdx")] tdx_enabled: bool,
-        #[cfg(feature = "sev_snp")] sev_snp_enabled: bool,
-        #[cfg(feature = "sev_snp")] mem_size: u64,
+        config: HypervisorVmConfig,
     ) -> Result<Arc<dyn hypervisor::Vm>> {
         hypervisor.check_required_extensions().unwrap();
-        let config = HypervisorVmConfig {
-            #[cfg(feature = "tdx")]
-            tdx_enabled,
-            #[cfg(feature = "sev_snp")]
-            sev_snp_enabled,
-            #[cfg(feature = "sev_snp")]
-            mem_size,
-        };
 
         let vm = hypervisor.create_vm(config).unwrap();