Skip to content

CKS: fix scaling failures for clusters without a default service offering #14138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Pearl1594 wants to merge 2 commits into
base: 4.22
Choose a base branch
from
+72 −38
Draft

CKS: fix scaling failures for clusters without a default service offering #14138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
21b76ea
CCKS: fix scaling failures for clusters without a default service off…
Pearl1594 Sep 10, 2026
779fa79
extract common code and check node-type offering if present during up…
Pearl1594 Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 62 additions & 22 deletions ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1398,6 +1398,16 @@ protected void ensureResourceLimitsForScale(final KubernetesClusterVO cluster,
}
}

public static long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
if (WORKER == nodeType) {
return kubernetesCluster.getNodeCount();
} else if (CONTROL == nodeType) {
return kubernetesCluster.getControlNodeCount();
} else if (ETCD == nodeType) {
return kubernetesCluster.getEtcdNodeCount();
}
return kubernetesCluster.getTotalNodeCount();
}

protected void validateServiceOfferingsForNodeTypesScale(Map<String, Long> map, Long defaultServiceOfferingId, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) {
for (String key : CLUSTER_NODES_TYPES_LIST) {
Expand All @@ -1408,11 +1418,10 @@ protected void validateServiceOfferingsForNodeTypesScale(Map<String, Long> map,
throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId);
}
checkServiceOfferingForNodesScale(serviceOffering, kubernetesCluster, clusterVersion);
Long nodeTypeOfferingId = getExistingServiceOfferingIdForNodeType(key, kubernetesCluster);
if (nodeTypeOfferingId == null) {
nodeTypeOfferingId = kubernetesCluster.getServiceOfferingId();
final ServiceOffering existingServiceOffering = getEffectiveServiceOfferingForNodeType(key, kubernetesCluster);
if (existingServiceOffering == null) {
continue;
}
final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(nodeTypeOfferingId);
if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() ||
serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) {
logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s",
Expand All @@ -1433,6 +1442,33 @@ private Long getExistingServiceOfferingIdForNodeType(String key, KubernetesClust
return kubernetesCluster.getServiceOfferingId();
}

// Resolves the offering backing a node type, falling back to the legacy cluster-wide offering; null if the cluster has no nodes of this type.
protected ServiceOffering getEffectiveServiceOfferingForNodeType(String key, KubernetesClusterVO kubernetesCluster) {
if (getNodeCountForType(KubernetesClusterNodeType.valueOf(key), kubernetesCluster) <= 0) {
return null;
}
Long offeringId = getExistingServiceOfferingIdForNodeType(key, kubernetesCluster);
if (offeringId == null) {
offeringId = kubernetesCluster.getServiceOfferingId();
}
if (offeringId == null) {
return null;
}
return serviceOfferingDao.findByIdIncludingRemoved(offeringId);
}

// Effective offerings for every node type actually present on the cluster.
protected List<ServiceOffering> getEffectiveServiceOfferingsForCluster(KubernetesClusterVO kubernetesCluster) {
List<ServiceOffering> offerings = new ArrayList<>();
for (String key : CLUSTER_NODES_TYPES_LIST) {
ServiceOffering offering = getEffectiveServiceOfferingForNodeType(key, kubernetesCluster);
if (offering != null) {
offerings.add(offering);
}
}
return offerings;
}

protected void checkServiceOfferingForNodesScale(ServiceOffering serviceOffering, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) {
if (serviceOffering.isDynamic()) {
throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName()));
Expand Down Expand Up @@ -1497,17 +1533,19 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster
throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster : %s",
kubernetesCluster.getName()));
}
final ServiceOffering serviceOffering = serviceOfferingDao.findByIdIncludingRemoved(kubernetesCluster.getServiceOfferingId());
if (serviceOffering == null) {
final List<ServiceOffering> effectiveServiceOfferings = getEffectiveServiceOfferingsForCluster(kubernetesCluster);
if (CollectionUtils.isEmpty(effectiveServiceOfferings)) {
throw new CloudRuntimeException(String.format("Invalid service offering associated with Kubernetes cluster : %s", kubernetesCluster.getName()));
}
if (serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d vCPUs while associated service offering : %s offers only %d vCPUs",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumCpu(), serviceOffering.getName(), serviceOffering.getCpu()));
}
if (serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d MB RAM while associated service offering : %s offers only %d MB RAM",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumRamSize(), serviceOffering.getName(), serviceOffering.getRamSize()));
for (ServiceOffering serviceOffering : effectiveServiceOfferings) {
if (serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d vCPUs while associated service offering : %s offers only %d vCPUs",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumCpu(), serviceOffering.getName(), serviceOffering.getCpu()));
}
if (serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d MB RAM while associated service offering : %s offers only %d MB RAM",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumRamSize(), serviceOffering.getName(), serviceOffering.getRamSize()));
}
}
// Check upgradeVersion is either patch upgrade or immediate minor upgrade
try {
Expand Down Expand Up @@ -2196,18 +2234,20 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou
protected Map<String, ServiceOffering> createNodeTypeToServiceOfferingMap(Map<String, Long> idsMapping,
Long serviceOfferingId, KubernetesClusterVO kubernetesCluster) {
Map<String, ServiceOffering> map = new HashMap<>();
if (MapUtils.isEmpty(idsMapping)) {
ServiceOfferingVO offering = serviceOfferingId != null ?
serviceOfferingDao.findById(serviceOfferingId) :
serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
map.put(DEFAULT.name(), offering);
if (MapUtils.isNotEmpty(idsMapping)) {
for (String key : CLUSTER_NODES_TYPES_LIST) {
if (idsMapping.containsKey(key)) {
map.put(key, serviceOfferingDao.findById(idsMapping.get(key)));
}
}
return map;
}
for (String key : CLUSTER_NODES_TYPES_LIST) {
if (!idsMapping.containsKey(key)) {
continue;
Long defaultOfferingId = serviceOfferingId != null ? serviceOfferingId : kubernetesCluster.getServiceOfferingId();
if (defaultOfferingId != null) {
ServiceOfferingVO offering = serviceOfferingDao.findById(defaultOfferingId);
if (offering != null) {
map.put(DEFAULT.name(), offering);
}
map.put(key, serviceOfferingDao.findById(idsMapping.get(key)));
}
return map;
}
Expand Down
26 changes: 10 additions & 16 deletions ...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ protected Pair<Long, Long> calculateNewClusterCountAndCapacity(Long newWorkerSiz
cores = serviceOffering.getCpu() * totalClusterSize;
memory = serviceOffering.getRamSize() * totalClusterSize;
} else {
long nodeCount = getNodeCountForType(nodeType, kubernetesCluster);
long nodeCount = KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster);
Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster);
if (existingOfferingId == null) {
existingOfferingId = serviceOffering.getId();
Expand All @@ -243,17 +243,6 @@ protected Pair<Long, Long> calculateNewClusterCountAndCapacity(Long newWorkerSiz
return new Pair<>(cores, memory);
}

private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
if (WORKER == nodeType) {
return kubernetesCluster.getNodeCount();
} else if (CONTROL == nodeType) {
return kubernetesCluster.getControlNodeCount();
} else if (ETCD == nodeType) {
return kubernetesCluster.getEtcdNodeCount();
}
return kubernetesCluster.getTotalNodeCount();
}

protected Pair<Long, Long> calculateClusterNewCapacity(KubernetesCluster kubernetesCluster,
Pair<Long, Long> previousNodeTypeCapacity,
Pair<Long, Long> newNodeTypeCapacity) {
Expand Down Expand Up @@ -337,17 +326,19 @@ private void validateKubernetesClusterScaleSizeParameters() throws CloudRuntimeE
// Check capacity and transition state
final long newVmRequiredCount = clusterSize - originalClusterSize;
final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
if (clusterServiceOffering == null) {
final ServiceOffering existingWorkerOffering = serviceOfferingDao.findById(kubernetesCluster.getWorkerNodeServiceOfferingId());
final ServiceOffering effectiveOffering = existingWorkerOffering != null ? existingWorkerOffering : clusterServiceOffering;
if (effectiveOffering == null) {
logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s, cluster service offering not found", kubernetesCluster.getName()));
}
if (newVmRequiredCount > 0) {
final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId());
try {
if (originalState.equals(KubernetesCluster.State.Running)) {
plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
plan(newVmRequiredCount, zone, effectiveOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
} else {
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, effectiveOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
}
} catch (InsufficientCapacityException e) {
logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s in zone : %s, insufficient capacity", kubernetesCluster.getName(), zone.getName()));
Expand All @@ -370,7 +361,7 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType,
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
return;
}
final long size = getNodeCountForType(nodeType, kubernetesCluster);
final long size = KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster);
List<KubernetesClusterVmMapVO> vmList = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
Expand Down Expand Up @@ -610,6 +601,9 @@ public boolean scaleCluster() throws CloudRuntimeException {
private ServiceOffering getExistingServiceOfferingForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster);
if (existingOfferingId == null) {
if (KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster) <= 0) {
return null;
}
logAndThrow(Level.ERROR, String.format("The Kubernetes cluster %s does not have a service offering set for node type %s",
kubernetesCluster.getName(), nodeType.name()));
}
Expand Down
Loading