Skip to content

DRS: honour anti-affinity when planning and executing migrations #14108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bhouse-nexthop wants to merge 8 commits into
base: 4.22
Choose a base branch
from
Open

DRS: honour anti-affinity when planning and executing migrations #14108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1faba7d
Fix unreachable and NPE-prone branch in host anti-affinity
bhouse-nexthop Sep 9, 2026
494ead4
Honor planned placements in host anti-affinity
bhouse-nexthop Sep 9, 2026
65d4fce
Honor planned placements in non-strict host affinity
bhouse-nexthop Sep 9, 2026
3459b70
Re-check affinity before executing each DRS migration
bhouse-nexthop Sep 9, 2026
b7ed756
Make the DRS affinity check correct
bhouse-nexthop Sep 9, 2026
192bab5
Address the rest of the DRS anti-affinity review
bhouse-nexthop Sep 9, 2026
6019289
Re-check every migration, not only VMs in an affinity group
bhouse-nexthop Sep 10, 2026
f0ebe8c
Drop the duplicate getVmIdVmMap from HostAffinityProcessor
bhouse-nexthop Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import com.cloud.vm.VirtualMachineProfile;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class AffinityProcessorBase extends AdapterBase implements AffinityGroupProcessor {

Expand All @@ -41,6 +43,23 @@ public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList a

}


/**
* Indexes placements supplied by the caller. Callers such as DRS build a plan of several moves
* in memory and persist it only at the end, so during planning the database still shows the old
* host for every VM the plan has already moved.
*/
protected Map<Long, VirtualMachine> getVmIdVmMap(List<VirtualMachine> vmList) {
Comment thread
bhouse-nexthop marked this conversation as resolved.
Map<Long, VirtualMachine> vmIdVmMap = new HashMap<>();
if (vmList == null) {
return vmIdVmMap;
}
for (VirtualMachine vm : vmList) {
vmIdVmMap.put(vm.getId(), vm);
}
return vmIdVmMap;
}

@Override
public String getType() {
return _type;
Expand Down
9 changes: 0 additions & 9 deletions ...ors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.apache.cloudstack.affinity;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -121,14 +120,6 @@ protected Set<Long> getHostIdSet(List<Long> vmIds, List<VirtualMachine> vmList)
return hostIds;
}

protected Map<Long, VirtualMachine> getVmIdVmMap(List<VirtualMachine> vmList) {
Map<Long, VirtualMachine> vmIdVmMap = new HashMap<>();
for (VirtualMachine vm : vmList) {
vmIdVmMap.put(vm.getId(), vm);
}
return vmIdVmMap;
}

/**
* Get preferred host ids list from the affinity group VMs
*/
Expand Down
58 changes: 41 additions & 17 deletions ...anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.cloudstack.affinity;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -59,7 +60,7 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements
protected AffinityGroupDao _affinityGroupDao;
@Inject
protected AffinityGroupVMMapDao _affinityGroupVMMapDao;
private int _vmCapacityReleaseInterval;
protected int _vmCapacityReleaseInterval;
@Inject
protected ConfigurationDao _configDao;

Expand All @@ -82,14 +83,26 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
_affinityGroupDao.listByIds(affinityGroupIds, true);
}
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
processAffinityGroup(vmGroupMapping, avoid, vm);
processAffinityGroup(vmGroupMapping, avoid, vm, vmList);
}
}
});

}

protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) {
processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList());
}

/**
* Applies anti-affinity for one group.
*
* @param vmList
* placements to honour in preference to what the database says. DRS builds a plan of
* several migrations in memory and only persists it later, so during plan generation
* the database still shows the old host for every VM the plan has already moved.
*/
protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm, List<VirtualMachine> vmList) {
if (vmGroupMapping != null) {
AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());

Expand All @@ -100,24 +113,35 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude
List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
groupVMIds.remove(vm.getId());

Map<Long, VirtualMachine> plannedVms = getVmIdVmMap(vmList);

for (Long groupVMId : groupVMIds) {
VirtualMachine plannedVm = plannedVms.get(groupVMId);
if (plannedVm != null && plannedVm.getHostId() != null) {
avoid.addHost(plannedVm.getHostId());
logger.debug("Added host {} to avoid set, since VM {} is placed on the host by the plan being built",
plannedVm.getHostId(), plannedVm);
continue;
}
VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
if (groupVM != null && !groupVM.isRemoved()) {
if (groupVM.getHostId() != null) {
avoid.addHost(groupVM.getHostId());
if (logger.isDebugEnabled()) {
logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM);
}
}
} else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
avoid.addHost(groupVM.getLastHostId());
if (logger.isDebugEnabled()) {
logger.debug("Added host {} to avoid set, since VM {} is present on the host, in Stopped state but has reserved capacity", groupVM.getLastHostId(), groupVM);
}
}
if (groupVM == null || groupVM.isRemoved()) {
continue;
}
avoidHostOfVmInAffinityGroup(avoid, groupVM);
}
}
}

protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) {
if (groupVM.getHostId() != null) {
avoid.addHost(groupVM.getHostId());
logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM);
} else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
avoid.addHost(groupVM.getLastHostId());
logger.debug("Added host {} to avoid set, since VM {} is in {} state on the host but still has reserved capacity",
groupVM.getLastHostId(), groupVM, groupVM.getState());
}
}
}
Expand Down
233 changes: 233 additions & 0 deletions ...-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.affinity;

import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.utils.DateUtil;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;

@RunWith(JUnit4.class)
public class HostAntiAffinityProcessorTest {

private static final long AFFINITY_GROUP_ID = 2L;
private static final long VM_ID = 3L;
private static final long GROUP_VM_ID = 1L;
private static final long HOST_ID = 10L;
private static final long LAST_HOST_ID = 11L;
private static final long PLANNED_HOST_ID = 12L;
private static final int CAPACITY_RELEASE_INTERVAL = 3600;

@Mock
AffinityGroupDao _affinityGroupDao;

@Mock
AffinityGroupVMMapDao _affinityGroupVMMapDao;

@Mock
VMInstanceDao _vmInstanceDao;

@Spy
@InjectMocks
HostAntiAffinityProcessor processor = new HostAntiAffinityProcessor();

@Mock
VirtualMachine vm;

@Mock
VMInstanceVO groupVM;

@Mock
AffinityGroupVO affinityGroupVO;

@Mock
AffinityGroupVMMapVO vmGroupMapping;

private ExcludeList avoid;

@Before
public void setUp() {
MockitoAnnotations.openMocks(this);
processor._vmCapacityReleaseInterval = CAPACITY_RELEASE_INTERVAL;
avoid = new ExcludeList();

when(vm.getId()).thenReturn(VM_ID);
when(vmGroupMapping.getAffinityGroupId()).thenReturn(AFFINITY_GROUP_ID);
when(_affinityGroupDao.findById(AFFINITY_GROUP_ID)).thenReturn(affinityGroupVO);
when(affinityGroupVO.getId()).thenReturn(AFFINITY_GROUP_ID);
when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID))
.thenReturn(new ArrayList<>(Arrays.asList(GROUP_VM_ID, VM_ID)));
}

private boolean avoids(long hostId) {
return avoid.getHostsToAvoid() != null && avoid.getHostsToAvoid().contains(hostId);
}

@Test
public void testRunningGroupVmHostIsAvoided() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(HOST_ID);

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertTrue(avoids(HOST_ID));
}

@Test
public void testStoppedGroupVmWithReservedCapacityLastHostIsAvoided() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(null);
when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped);
when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID);
when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime());

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertTrue(avoids(LAST_HOST_ID));
}

@Test
public void testStoppedGroupVmPastReleaseIntervalIsNotAvoided() {
Date wellPast = new Date(DateUtil.currentGMTTime().getTime() - (CAPACITY_RELEASE_INTERVAL + 60) * 1000L);
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(null);
when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped);
when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID);
when(groupVM.getUpdateTime()).thenReturn(wellPast);

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertFalse(avoids(LAST_HOST_ID));
}

@Test
public void testMissingGroupVmIsSkipped() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(null);

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty());
}

@Test
public void testRemovedGroupVmIsSkipped() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(true);
// a removed VM is not running anywhere, so neither of its hosts should be avoided
lenient().when(groupVM.getHostId()).thenReturn(HOST_ID);
lenient().when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped);
lenient().when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID);
lenient().when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime());

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertFalse(avoids(HOST_ID));
assertFalse(avoids(LAST_HOST_ID));
}

@Test
public void testStartingGroupVmWithReservedCapacityLastHostIsAvoided() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(null);
when(groupVM.getState()).thenReturn(VirtualMachine.State.Starting);
when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID);
when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime());

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertTrue(avoids(LAST_HOST_ID));
}

@Test
public void testPlannedHostWinsOverStaleDatabaseHost() {
VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class);
when(plannedVm.getId()).thenReturn(GROUP_VM_ID);
when(plannedVm.getHostId()).thenReturn(PLANNED_HOST_ID);

// the database still shows the pre-migration host
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(HOST_ID);

processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm));

assertTrue(avoids(PLANNED_HOST_ID));
assertFalse(avoids(HOST_ID));
}

@Test
public void testPlannedVmWithoutHostFallsBackToDatabase() {
VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class);
when(plannedVm.getId()).thenReturn(GROUP_VM_ID);
when(plannedVm.getHostId()).thenReturn(null);

when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(HOST_ID);

processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm));

assertTrue(avoids(HOST_ID));
}

@Test
public void testEmptyVmListBehavesAsBefore() {
when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM);
when(groupVM.isRemoved()).thenReturn(false);
when(groupVM.getHostId()).thenReturn(HOST_ID);

processor.processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList());

assertTrue(avoids(HOST_ID));
}

@Test
public void testVmIsNotAvoidedAgainstItself() {
List<Long> ids = new ArrayList<>(Arrays.asList(VM_ID));
when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)).thenReturn(ids);

processor.processAffinityGroup(vmGroupMapping, avoid, vm);

assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty());
}
}
Loading
Loading