Skip to content

Commit 0f26d5a

Browse files
kelvenykelveny
committed
Add skeleton to support out-of-band sync and recovery from left-over transitional VM states
1 parent 2892469 commit 0f26d5a

4 files changed

Lines changed: 118 additions & 190 deletions

File tree

‎engine/schema/src/org/apache/cloudstack/vm/jobs/VmWorkJobDao.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323

2424
import com.cloud.utils.db.GenericDao;
2525
import com.cloud.vm.VirtualMachine;
26-
import com.cloud.vm.VirtualMachine.Type;
2726

2827
public interface VmWorkJobDao extends GenericDao<VmWorkJobVO, Long> {
2928
VmWorkJobVO findPendingWorkJob(VirtualMachine.Type type, long instanceId);

‎server/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 108 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import org.apache.cloudstack.framework.jobs.AsyncJobConstants;
4242
import org.apache.cloudstack.framework.jobs.AsyncJobManager;
4343
import org.apache.cloudstack.framework.messagebus.MessageBus;
44+
import org.apache.cloudstack.framework.messagebus.MessageDispatcher;
45+
import org.apache.cloudstack.framework.messagebus.MessageHandler;
4446
import org.apache.cloudstack.messagebus.TopicConstants;
4547
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
4648
import org.apache.cloudstack.vm.jobs.VmWorkJobDao;
@@ -238,6 +240,9 @@ public void setHostAllocators(List<HostAllocator> hostAllocators) {
238240
protected ClusterDetailsDao _clusterDetailsDao;
239241
@Inject
240242
protected UserVmDetailsDao _uservmDetailsDao;
243+
244+
@Inject
245+
protected VMInstanceDao _instanceDao;
241246

242247
@Inject
243248
protected ConfigurationDao _configDao;
@@ -427,14 +432,17 @@ public void advanceExpunge(String vmUuid) throws ResourceUnavailableException, O
427432

428433
_vmDao.remove(vm.getId());
429434
}
435+
436+
430437

431438
@Override
432439
public boolean start() {
433440
_executor.scheduleAtFixedRate(new CleanupTask(), _cleanupInterval, _cleanupInterval, TimeUnit.SECONDS);
434441
cancelWorkItems(_nodeId);
442+
435443
return true;
436444
}
437-
445+
438446
@Override
439447
public boolean stop() {
440448
return true;
@@ -469,6 +477,9 @@ public boolean configure(String name, Map<String, Object> xmlParams) throws Conf
469477
.and(rvsEntity.getInstanceId(), SearchCriteria.Op.EQ, "instance")
470478
.and(rvsEntity.getDeviceId(), SearchCriteria.Op.EQ).values(0)
471479
.done();
480+
481+
482+
_messageBus.subscribe(TopicConstants.VM_POWER_STATE, MessageDispatcher.getDispatcher(this));
472483

473484
return true;
474485
}
@@ -1311,176 +1322,6 @@ public void orchestrateStop(String vmUuid, boolean forced) throws AgentUnavailab
13111322
}
13121323
}
13131324

1314-
/*
1315-
@Override
1316-
public boolean advanceStop(String vmUuid, boolean forced, User user, Account account) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException {
1317-
State state = vm.getState();
1318-
if (state == State.Stopped) {
1319-
if (s_logger.isDebugEnabled()) {
1320-
s_logger.debug("VM is already stopped: " + vm);
1321-
}
1322-
return true;
1323-
}
1324-
1325-
if (state == State.Destroyed || state == State.Expunging || state == State.Error) {
1326-
if (s_logger.isDebugEnabled()) {
1327-
s_logger.debug("Stopped called on " + vm + " but the state is " + state);
1328-
}
1329-
return true;
1330-
}
1331-
// grab outstanding work item if any
1332-
VmWorkJobVO work = _workDao.findByOutstandingWork(vm.getId(), vm.getState());
1333-
if (work != null) {
1334-
if (s_logger.isDebugEnabled()) {
1335-
s_logger.debug("Found an outstanding work item for this vm " + vm + " with state:" + vm.getState() + ", work id:" + work.getId());
1336-
}
1337-
}
1338-
Long hostId = vm.getHostId();
1339-
if (hostId == null) {
1340-
if (!forced) {
1341-
if (s_logger.isDebugEnabled()) {
1342-
s_logger.debug("HostId is null but this is not a forced stop, cannot stop vm " + vm + " with state:" + vm.getState());
1343-
}
1344-
return false;
1345-
}
1346-
try {
1347-
stateTransitTo(vm, Event.AgentReportStopped, null, null);
1348-
} catch (NoTransitionException e) {
1349-
s_logger.warn(e.getMessage());
1350-
}
1351-
// mark outstanding work item if any as done
1352-
if (work != null) {
1353-
if (s_logger.isDebugEnabled()) {
1354-
s_logger.debug("Updating work item to Done, id:" + work.getId());
1355-
}
1356-
work.setStep(Step.Done);
1357-
_workDao.update(work.getId(), work);
1358-
}
1359-
return true;
1360-
}
1361-
1362-
VirtualMachineGuru<T> vmGuru = getVmGuru(vm);
1363-
VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
1364-
1365-
try {
1366-
if (!stateTransitTo(vm, Event.StopRequested, vm.getHostId())) {
1367-
throw new ConcurrentOperationException("VM is being operated on.");
1368-
}
1369-
} catch (NoTransitionException e1) {
1370-
if (!forced) {
1371-
throw new CloudRuntimeException("We cannot stop " + vm + " when it is in state " + vm.getState());
1372-
}
1373-
boolean doCleanup = false;
1374-
if (s_logger.isDebugEnabled()) {
1375-
s_logger.debug("Unable to transition the state but we're moving on because it's forced stop");
1376-
}
1377-
if (state == State.Starting || state == State.Migrating) {
1378-
if (work != null) {
1379-
doCleanup = true;
1380-
} else {
1381-
if (s_logger.isDebugEnabled()) {
1382-
s_logger.debug("Unable to cleanup VM: " + vm + " ,since outstanding work item is not found");
1383-
}
1384-
throw new CloudRuntimeException("Work item not found, We cannot stop " + vm + " when it is in state " + vm.getState());
1385-
}
1386-
} else if (state == State.Stopping) {
1387-
doCleanup = true;
1388-
}
1389-
1390-
if (doCleanup) {
1391-
if (cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, forced, user, account)) {
1392-
try {
1393-
if (s_logger.isDebugEnabled()) {
1394-
s_logger.debug("Updating work item to Done, id:" + work.getId());
1395-
}
1396-
return changeState(vm, Event.AgentReportStopped, null, work, Step.Done);
1397-
} catch (NoTransitionException e) {
1398-
s_logger.warn("Unable to cleanup " + vm);
1399-
return false;
1400-
}
1401-
} else {
1402-
if (s_logger.isDebugEnabled()) {
1403-
s_logger.debug("Failed to cleanup VM: " + vm);
1404-
}
1405-
throw new CloudRuntimeException("Failed to cleanup " + vm + " , current state " + vm.getState());
1406-
}
1407-
}
1408-
}
1409-
1410-
if (vm.getState() != State.Stopping) {
1411-
throw new CloudRuntimeException("We cannot proceed with stop VM " + vm + " since it is not in 'Stopping' state, current state: " + vm.getState());
1412-
}
1413-
1414-
vmGuru.prepareStop(profile);
1415-
1416-
StopCommand stop = new StopCommand(vm);
1417-
boolean stopped = false;
1418-
StopAnswer answer = null;
1419-
try {
1420-
answer = (StopAnswer) _agentMgr.send(vm.getHostId(), stop);
1421-
stopped = answer.getResult();
1422-
if (!stopped) {
1423-
throw new CloudRuntimeException("Unable to stop the virtual machine due to " + answer.getDetails());
1424-
}
1425-
vmGuru.finalizeStop(profile, answer);
1426-
1427-
} catch (AgentUnavailableException e) {
1428-
s_logger.warn("Unable to stop vm, agent unavailable: " + e.toString());
1429-
} catch (OperationTimedoutException e) {
1430-
s_logger.warn("Unable to stop vm, operation timed out: " + e.toString());
1431-
} finally {
1432-
if (!stopped) {
1433-
if (!forced) {
1434-
s_logger.warn("Unable to stop vm " + vm);
1435-
try {
1436-
stateTransitTo(vm, Event.OperationFailed, vm.getHostId());
1437-
} catch (NoTransitionException e) {
1438-
s_logger.warn("Unable to transition the state " + vm);
1439-
}
1440-
return false;
1441-
} else {
1442-
s_logger.warn("Unable to actually stop " + vm + " but continue with release because it's a force stop");
1443-
vmGuru.finalizeStop(profile, answer);
1444-
}
1445-
}
1446-
}
1447-
1448-
if (s_logger.isDebugEnabled()) {
1449-
s_logger.debug(vm + " is stopped on the host. Proceeding to release resource held.");
1450-
}
1451-
1452-
try {
1453-
_networkMgr.release(profile, forced);
1454-
s_logger.debug("Successfully released network resources for the vm " + vm);
1455-
} catch (Exception e) {
1456-
s_logger.warn("Unable to release some network resources.", e);
1457-
}
1458-
1459-
try {
1460-
if (vm.getHypervisorType() != HypervisorType.BareMetal) {
1461-
this.volumeMgr.release(profile);
1462-
s_logger.debug("Successfully released storage resources for the vm " + vm);
1463-
}
1464-
} catch (Exception e) {
1465-
s_logger.warn("Unable to release storage resources.", e);
1466-
}
1467-
1468-
try {
1469-
if (work != null) {
1470-
if (s_logger.isDebugEnabled()) {
1471-
s_logger.debug("Updating the outstanding work item to Done, id:" + work.getId());
1472-
}
1473-
work.setStep(Step.Done);
1474-
_workDao.update(work.getId(), work);
1475-
}
1476-
1477-
return stateTransitTo(vm, Event.OperationSucceeded, null);
1478-
} catch (NoTransitionException e) {
1479-
s_logger.warn(e.getMessage());
1480-
return false;
1481-
}
1482-
}
1483-
*/
14841325
private void setStateMachine() {
14851326
_stateMachine = VirtualMachine.State.getStateMachine();
14861327
}
@@ -3569,5 +3410,101 @@ public boolean reConfigureVm(VirtualMachine vm, ServiceOffering oldServiceOfferi
35693410
return false;
35703411

35713412
}
3413+
3414+
@MessageHandler(topic=TopicConstants.VM_POWER_STATE)
3415+
private void HandlePownerStateReport(Object target, String subject, String senderAddress, Object args) {
3416+
assert(args != null);
3417+
Long vmId = (Long)args;
3418+
3419+
List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(
3420+
VirtualMachine.Type.Instance, vmId);
3421+
if(pendingWorkJobs.size() == 0) {
3422+
// there is no pending operation job
3423+
VMInstanceVO vm = _instanceDao.findById(vmId);
3424+
if(vm != null) {
3425+
switch(vm.getPowerState()) {
3426+
case PowerOn :
3427+
HandlePowerOnReportWithNoPendingJobsOnVM(vm);
3428+
break;
3429+
3430+
case PowerOff :
3431+
HandlePowerOffReportWithNoPendingJobsOnVM(vm);
3432+
break;
3433+
3434+
// PowerUnknown shouldn't be reported, it is a derived
3435+
// VM power state from host state (host un-reachable
3436+
case PowerUnknown :
3437+
default :
3438+
assert(false);
3439+
break;
3440+
}
3441+
} else {
3442+
s_logger.warn("VM " + vmId + " no longer exists when processing VM state report");
3443+
}
3444+
} else {
3445+
// TODO, wake-up signalling
3446+
}
3447+
}
3448+
3449+
private void HandlePowerOnReportWithNoPendingJobsOnVM(VMInstanceVO vm) {
35723450

3451+
// TODO :
3452+
// 1) handle left-over transitional VM states
3453+
// 2) handle out of band VM live migration
3454+
// 3) handle out of sync stationary states, marking VM from Stopped to Running with
3455+
// alert messages
3456+
3457+
switch(vm.getState()) {
3458+
case Starting:
3459+
break;
3460+
3461+
case Running :
3462+
break;
3463+
3464+
case Stopping :
3465+
case Stopped :
3466+
break;
3467+
3468+
case Destroyed :
3469+
case Expunging :
3470+
break;
3471+
3472+
case Migrating :
3473+
break;
3474+
3475+
case Error :
3476+
default :
3477+
break;
3478+
}
3479+
}
3480+
3481+
private void HandlePowerOffReportWithNoPendingJobsOnVM(VMInstanceVO vm) {
3482+
3483+
// TODO :
3484+
// 1) handle left-over transitional VM states
3485+
// 2) handle out of sync stationary states, schedule force-stop to release resources
3486+
//
3487+
switch(vm.getState()) {
3488+
case Starting:
3489+
break;
3490+
3491+
case Running :
3492+
break;
3493+
3494+
case Stopping :
3495+
case Stopped :
3496+
break;
3497+
3498+
case Destroyed :
3499+
case Expunging :
3500+
break;
3501+
3502+
case Migrating :
3503+
break;
3504+
3505+
case Error :
3506+
default :
3507+
break;
3508+
}
3509+
}
35733510
}

‎server/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ public void processHostVmStatePingReport(long hostId, Map<String, PowerState> re
6666
}
6767

6868
private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> translatedInfo) {
69+
70+
6971
for(Map.Entry<Long, VirtualMachine.PowerState> entry : translatedInfo.entrySet()) {
7072

7173
if(s_logger.isDebugEnabled())
@@ -79,6 +81,11 @@ private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> tra
7981
_messageBus.publish(null, TopicConstants.VM_POWER_STATE, PublishScope.GLOBAL, entry.getKey());
8082
}
8183
}
84+
85+
//
86+
// TODO
87+
// 1) publish missing report (if VM is missing from host report) for KVM/XenServer
88+
//
8289
}
8390

8491
private Map<Long, VirtualMachine.PowerState> convertHostPingInfos(Map<String, PowerState> states) {
@@ -121,20 +128,5 @@ private Map<Long, VirtualMachine.PowerState> convertToInfos(Map<String, HostVmSt
121128

122129
private VMInstanceVO findVM(String vmName) {
123130
return _instanceDao.findVMByInstanceName(vmName);
124-
// Collection<VirtualMachineGuru> vmGurus = _vmMgr.getRegisteredGurus();
125-
//
126-
// for (VirtualMachineGuru vmGuru : vmGurus) {
127-
// VMInstanceVO vm = vmGuru.findByName(vmName);
128-
// if (vm != null)
129-
// return vm;
130-
//
131-
// Long id = vmGuru.convertToId(vmName);
132-
// if (id != null) {
133-
// vm = vmGuru.findById(id);
134-
// if(vm != null)
135-
// return vm;
136-
// }
137-
// }
138-
// return null;
139131
}
140132
}

‎server/src/org/apache/cloudstack/messagebus/TopicConstants.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818

1919
public interface TopicConstants {
2020
// VM power state messages on message bus
21-
public static final String VM_POWER_STATE = "vm.powerstate";
21+
public static final String VM_POWER_STATE = "vm.powerstate"; // args <Long> vmid
2222

2323
// job messages on message bus
24-
public static final String JOB_HEARTBEAT = "job.heartbeat";
25-
public static final String JOB_STATE = "job.state";
24+
public static final String JOB_HEARTBEAT = "job.heartbeat"; // args <Long> jobid
25+
public static final String JOB_STATE = "job.state"; // args <Long> jobid
2626
}

0 commit comments

Comments
 (0)