Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion arch/src/x86_64/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub mod tdx;

// While modern architectures support more than 255 CPUs via x2APIC,
// legacy devices such as mptable support at most 254 CPUs.
pub(crate) const MAX_SUPPORTED_CPUS_LEGACY: u32 = 254;
pub const MAX_SUPPORTED_CPUS_LEGACY: u32 = 254;

// CPUID feature bits
#[cfg(feature = "kvm")]
Expand Down
30 changes: 25 additions & 5 deletions vmm/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ use crate::vm_config::*;
const MAX_NUM_PCI_SEGMENTS: u16 = 96;
const MAX_IOMMU_ADDRESS_WIDTH_BITS: u8 = 64;

#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
const MAX_SUPPORTED_CPUS: u32 = 8192;
#[cfg(not(all(feature = "kvm", target_arch = "x86_64")))]
const MAX_SUPPORTED_CPUS: u32 = 255;

/// Errors associated with VM configuration parameters.
#[derive(Debug, Error)]
pub enum Error {
Expand Down Expand Up @@ -182,6 +187,9 @@ pub enum ValidationError {
/// Max is less than boot
#[error("Max CPUs lower than boot CPUs")]
CpusMaxLowerThanBoot,
/// Too many CPUs.
#[error("Too many CPUs: specified {0} but {MAX_SUPPORTED_CPUS} is the limit")]
TooManyCpus(u32 /* specified CPUs */),
/// Missing file value for debug-console
#[cfg(target_arch = "x86_64")]
#[error("Path missing when using file mode for debug console")]
Expand Down Expand Up @@ -586,11 +594,11 @@ impl CpusConfig {
.add("features");
parser.parse(cpus).map_err(Error::ParseCpus)?;

let boot_vcpus: u8 = parser
let boot_vcpus: u32 = parser
.convert("boot")
.map_err(Error::ParseCpus)?
.unwrap_or(DEFAULT_VCPUS);
let max_vcpus: u8 = parser
let max_vcpus: u32 = parser
.convert("max")
.map_err(Error::ParseCpus)?
.unwrap_or(boot_vcpus);
Expand All @@ -605,7 +613,7 @@ impl CpusConfig {
.map_err(Error::ParseCpus)?
.unwrap_or(DEFAULT_MAX_PHYS_BITS);
let affinity = parser
.convert::<Tuple<u8, Vec<usize>>>("affinity")
.convert::<Tuple<u32, Vec<usize>>>("affinity")
.map_err(Error::ParseCpus)?
.map(|v| {
v.0.iter()
Expand Down Expand Up @@ -2147,7 +2155,7 @@ impl NumaConfig {
let cpus = parser
.convert::<IntegerList>("cpus")
.map_err(Error::ParseNuma)?
.map(|v| v.0.iter().map(|e| *e as u8).collect());
.map(|v| v.0.iter().map(|e| *e as u32).collect());
let distances = parser
.convert::<Tuple<u64, u64>>("distances")
.map_err(Error::ParseNuma)?
Expand Down Expand Up @@ -2523,6 +2531,15 @@ impl VmConfig {
return Err(ValidationError::CpusMaxLowerThanBoot);
}

if self.cpus.max_vcpus > MAX_SUPPORTED_CPUS {
// Note: historically, Cloud Hypervisor did not support more than 255(254 on x64)
// vCPUs: self.cpus.max_vcpus was of type u8, so 255 was the maximum;
// on x86_64, the legacy mptable/apic was limited to 254 CPUs.
//
// Now the limit is lifted on x86_64 targets. Other targests/archs: TBD.
return Err(ValidationError::TooManyCpus(self.cpus.max_vcpus));
}

if let Some(rate_limit_groups) = &self.rate_limit_groups {
for rate_limit_group in rate_limit_groups {
rate_limit_group.validate(self)?;
Expand Down Expand Up @@ -2614,7 +2631,10 @@ impl VmConfig {
return Err(ValidationError::CpuTopologyDiesPerPackage);
}

let total = t.threads_per_core * t.cores_per_die * t.dies_per_package * t.packages;
let total: u32 = (t.threads_per_core as u32)
* (t.cores_per_die as u32)
* (t.dies_per_package as u32)
* (t.packages as u32);
if total != self.cpus.max_vcpus {
return Err(ValidationError::CpuTopologyCount);
}
Expand Down
56 changes: 30 additions & 26 deletions vmm/src/cpu.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ pub enum Error {
#[error("Error setting up AMX")]
AmxEnable(#[source] anyhow::Error),

#[error("Maximum number of vCPUs exceeds host limit")]
MaximumVcpusExceeded,
#[error("Maximum number of vCPUs {0} exceeds host limit {1}")]
MaximumVcpusExceeded(u32, u32),

#[cfg(feature = "sev_snp")]
#[error("Failed to set sev control register")]
Expand Down Expand Up @@ -698,12 +698,16 @@ impl CpuManager {
numa_nodes: &NumaNodes,
#[cfg(feature = "sev_snp")] sev_snp_enabled: bool,
) -> Result<Arc<Mutex<CpuManager>>> {
if u32::from(config.max_vcpus) > hypervisor.get_max_vcpus() {
return Err(Error::MaximumVcpusExceeded);
if config.max_vcpus > hypervisor.get_max_vcpus() {
return Err(Error::MaximumVcpusExceeded(
config.max_vcpus,
hypervisor.get_max_vcpus(),
));
}

let mut vcpu_states = Vec::with_capacity(usize::from(config.max_vcpus));
vcpu_states.resize_with(usize::from(config.max_vcpus), VcpuState::default);
let max_vcpus = usize::try_from(config.max_vcpus).unwrap();
let mut vcpu_states = Vec::with_capacity(max_vcpus);
vcpu_states.resize_with(max_vcpus, VcpuState::default);
let hypervisor_type = hypervisor.hypervisor_type();
#[cfg(target_arch = "x86_64")]
let cpu_vendor = hypervisor.get_cpu_vendor();
Expand Down Expand Up @@ -755,7 +759,7 @@ impl CpuManager {
let affinity = if let Some(cpu_affinity) = config.affinity.as_ref() {
cpu_affinity
.iter()
.map(|a| (a.vcpu as u32, a.host_cpus.clone()))
.map(|a| (a.vcpu, a.host_cpus.clone()))
.collect()
} else {
BTreeMap::new()
Expand All @@ -781,7 +785,7 @@ impl CpuManager {
#[cfg(feature = "guest_debug")]
vm_debug_evt,
selected_cpu: 0,
vcpus: Vec::with_capacity(usize::from(config.max_vcpus)),
vcpus: Vec::with_capacity(max_vcpus),
seccomp_action,
vm_ops,
acpi_address: None,
Expand Down Expand Up @@ -895,10 +899,10 @@ impl CpuManager {
},
|t| {
(
t.threads_per_core.into(),
t.cores_per_die.into(),
t.dies_per_package.into(),
t.packages.into(),
t.threads_per_core,
t.cores_per_die,
t.dies_per_package,
t.packages,
)
},
);
Expand Down Expand Up @@ -934,7 +938,7 @@ impl CpuManager {
self.present_vcpus()
);

if desired_vcpus > self.config.max_vcpus as u32 {
if desired_vcpus > self.config.max_vcpus {
return Err(Error::DesiredVCpuCountExceedsMax);
}

Expand Down Expand Up @@ -1245,7 +1249,7 @@ impl CpuManager {
inserting: bool,
paused: Option<bool>,
) -> Result<()> {
if desired_vcpus > self.config.max_vcpus as u32 {
if desired_vcpus > self.config.max_vcpus {
return Err(Error::DesiredVCpuCountExceedsMax);
}

Expand Down Expand Up @@ -1418,11 +1422,11 @@ impl CpuManager {
}

pub fn boot_vcpus(&self) -> u32 {
self.config.boot_vcpus as u32
self.config.boot_vcpus
}

pub fn max_vcpus(&self) -> u32 {
self.config.max_vcpus as u32
self.config.max_vcpus
}

#[cfg(target_arch = "x86_64")]
Expand Down Expand Up @@ -1456,10 +1460,10 @@ impl CpuManager {
pub fn get_vcpu_topology(&self) -> Option<(u16, u16, u16, u16)> {
self.config.topology.clone().map(|t| {
(
t.threads_per_core.into(),
t.cores_per_die.into(),
t.dies_per_package.into(),
t.packages.into(),
t.threads_per_core,
t.cores_per_die,
t.dies_per_package,
t.packages,
)
})
}
Expand All @@ -1475,15 +1479,15 @@ impl CpuManager {
{
madt.write(36, arch::layout::APIC_START.0);

for cpu in 0..self.config.max_vcpus as u32 {
for cpu in 0..self.config.max_vcpus {
let x2apic_id = get_x2apic_id(cpu, self.get_vcpu_topology());

let lapic = LocalX2Apic {
r#type: acpi::ACPI_X2APIC_PROCESSOR,
length: 16,
processor_id: cpu,
apic_id: x2apic_id,
flags: if cpu < self.config.boot_vcpus as u32 {
flags: if cpu < self.config.boot_vcpus {
1 << MADT_CPU_ENABLE_FLAG
} else {
0
Expand Down Expand Up @@ -1535,8 +1539,8 @@ impl CpuManager {
r#type: acpi::ACPI_APIC_GENERIC_CPU_INTERFACE,
length: 80,
reserved0: 0,
cpu_interface_number: cpu as u32,
uid: cpu as u32,
cpu_interface_number: cpu,
uid: cpu,
flags: 1,
parking_version: 0,
performance_interrupt: 0,
Expand Down Expand Up @@ -2274,15 +2278,15 @@ impl Aml for CpuManager {
let uid = aml::Name::new("_CID".into(), &aml::EISAName::new("PNP0A05"));
// Bundle methods together under a common object
let methods = CpuMethods {
max_vcpus: self.config.max_vcpus as u32,
max_vcpus: self.config.max_vcpus,
dynamic: self.dynamic,
};
let mut cpu_data_inner: Vec<&dyn Aml> = vec![&hid, &uid, &methods];

#[cfg(target_arch = "x86_64")]
let topology = self.get_vcpu_topology();
let mut cpu_devices = Vec::new();
for cpu_id in 0..(self.config.max_vcpus as u32) {
for cpu_id in 0..self.config.max_vcpus {
let proximity_domain = *self.proximity_domain_per_cpu.get(&cpu_id).unwrap_or(&0);
let cpu_device = Cpu {
cpu_id,
Expand Down
2 changes: 1 addition & 1 deletion vmm/src/device_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1694,7 +1694,7 @@ impl DeviceManager {
) -> DeviceManagerResult<Arc<Mutex<dyn InterruptController>>> {
let interrupt_controller: Arc<Mutex<gic::Gic>> = Arc::new(Mutex::new(
gic::Gic::new(
self.config.lock().unwrap().cpus.boot_vcpus as u32,
self.config.lock().unwrap().cpus.boot_vcpus,
Arc::clone(&self.msi_interrupt_manager),
self.address_manager.vm.clone(),
)
Expand Down
9 changes: 8 additions & 1 deletion vmm/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ use anyhow::anyhow;
#[cfg(feature = "dbus_api")]
use api::dbus::{DBusApiOptions, DBusApiShutdownChannels};
use api::http::HttpApiHandle;
#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
use arch::x86_64::MAX_SUPPORTED_CPUS_LEGACY;
use console_devices::{pre_create_console_devices, ConsoleInfo};
use landlock::LandlockError;
use libc::{tcsetattr, termios, EFD_NONBLOCK, SIGINT, SIGTERM, TCSANOW};
Expand Down Expand Up @@ -888,6 +890,11 @@ impl Vmm {
))
})?;

#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
if config.lock().unwrap().max_apic_id() > MAX_SUPPORTED_CPUS_LEGACY {
vm.enable_x2apic_api().unwrap();
}

let phys_bits =
vm::physical_bits(&self.hypervisor, config.lock().unwrap().cpus.max_phys_bits);

Expand Down Expand Up @@ -1822,7 +1829,7 @@ impl RequestHandler for Vmm {
} else {
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
if let Some(desired_vcpus) = desired_vcpus {
config.cpus.boot_vcpus = desired_vcpus.try_into().unwrap();
config.cpus.boot_vcpus = desired_vcpus;
}
if let Some(desired_ram) = desired_ram {
config.memory.size = desired_ram;
Expand Down
13 changes: 10 additions & 3 deletions vmm/src/vm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ use anyhow::anyhow;
use arch::layout::{KVM_IDENTITY_MAP_START, KVM_TSS_START};
#[cfg(feature = "tdx")]
use arch::x86_64::tdx::TdvfSection;
#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
use arch::x86_64::MAX_SUPPORTED_CPUS_LEGACY;
#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
use arch::PciSpaceInfo;
use arch::{get_host_cpu_phys_bits, EntryPoint, NumaNode, NumaNodes};
Expand Down Expand Up @@ -944,7 +946,7 @@ impl Vm {
}

if let Some(cpus) = &config.cpus {
node.cpus.extend(cpus.iter().map(|cpu| *cpu as u32));
node.cpus.extend(cpus);
}

if let Some(pci_segments) = &config.pci_segments {
Expand Down Expand Up @@ -1022,6 +1024,11 @@ impl Vm {
vm_config.lock().unwrap().memory.total_size(),
)?;

#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
if vm_config.lock().unwrap().max_apic_id() > MAX_SUPPORTED_CPUS_LEGACY {
vm.enable_x2apic_api().unwrap();
}

let phys_bits = physical_bits(&hypervisor, vm_config.lock().unwrap().cpus.max_phys_bits);

let memory_manager = if let Some(snapshot) =
Expand Down Expand Up @@ -1655,7 +1662,7 @@ impl Vm {
.notify_hotplug(AcpiNotificationFlags::CPU_DEVICES_CHANGED)
.map_err(Error::DeviceManager)?;
}
self.config.lock().unwrap().cpus.boot_vcpus = desired_vcpus.try_into().unwrap();
self.config.lock().unwrap().cpus.boot_vcpus = desired_vcpus;
}

if let Some(desired_memory) = desired_memory {
Expand Down Expand Up @@ -2709,7 +2716,7 @@ impl Vm {
&mut self,
destination_url: &str,
) -> std::result::Result<DumpState, GuestDebuggableError> {
let nr_cpus = self.config.lock().unwrap().cpus.boot_vcpus as u32;
let nr_cpus = self.config.lock().unwrap().cpus.boot_vcpus;
let elf_note_size = self.get_note_size(NoteDescType::ElfAndVmm, nr_cpus) as isize;
let mut elf_phdr_num = 1;
let elf_sh_info = 0;
Expand Down
Loading
Loading