Skip to content

Commit 0bfc817

Browse files
kelvenykelveny
committed
Handle transitional states across management server restart
1 parent e2edae1 commit 0bfc817

4 files changed

Lines changed: 123 additions & 98 deletions

File tree

‎api/src/com/cloud/vm/VirtualMachine.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,12 @@ public static StateMachine2<State, VirtualMachine.Event, VirtualMachine> getStat
108108
s_fsm.addTransition(State.Stopping, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
109109
s_fsm.addTransition(State.Stopped, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
110110
s_fsm.addTransition(State.Running, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
111-
112111
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
112+
113+
s_fsm.addTransition(State.Starting, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
114+
s_fsm.addTransition(State.Stopping, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
115+
s_fsm.addTransition(State.Running, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
116+
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
113117
}
114118

115119
public static boolean isVmStarted(State oldState, Event e, State newState) {

‎engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 110 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -457,12 +457,12 @@ public void advanceExpunge(String vmUuid) throws ResourceUnavailableException, O
457457
_vmDao.remove(vm.getId());
458458
}
459459

460-
461-
462460
@Override
463461
public boolean start() {
464462
_executor.scheduleAtFixedRate(new TransitionTask(), _pingInterval.value(), _pingInterval.value(), TimeUnit.SECONDS);
465463
_executor.scheduleAtFixedRate(new CleanupTask(), _pingInterval.value()*2, _pingInterval.value()*2, TimeUnit.SECONDS);
464+
465+
// cancel jobs left-over from last run
466466
cancelWorkItems(_nodeId);
467467

468468
return true;
@@ -1801,42 +1801,42 @@ public VirtualMachineTO toVmTO(VirtualMachineProfile profile) {
18011801

18021802
protected void cancelWorkItems(long nodeId) {
18031803
/*
1804-
GlobalLock scanLock = GlobalLock.getInternLock("vmmgr.cancel.workitem");
1804+
GlobalLock scanLock = GlobalLock.getInternLock("vmmgr.cancel.workitem");
18051805
1806-
try {
1807-
if (scanLock.lock(3)) {
1808-
try {
1809-
List<VmWorkJobVO> works = _workDao.listWorkInProgressFor(nodeId);
1810-
for (VmWorkJobVO work : works) {
1811-
s_logger.info("Handling unfinished work item: " + work);
1812-
try {
1813-
VMInstanceVO vm = _vmDao.findById(work.getInstanceId());
1814-
if (vm != null) {
1815-
if (work.getType() == State.Starting) {
1816-
_haMgr.scheduleRestart(vm, true);
1817-
work.setManagementServerId(_nodeId);
1818-
_workDao.update(work.getId(), work);
1819-
} else if (work.getType() == State.Stopping) {
1820-
_haMgr.scheduleStop(vm, vm.getHostId(), WorkType.CheckStop);
1821-
work.setManagementServerId(_nodeId);
1822-
_workDao.update(work.getId(), work);
1823-
} else if (work.getType() == State.Migrating) {
1824-
_haMgr.scheduleMigration(vm);
1825-
work.setStep(Step.Done);
1826-
_workDao.update(work.getId(), work);
1827-
}
1806+
try {
1807+
if (scanLock.lock(3)) {
1808+
try {
1809+
List<VmWorkJobVO> works = _workDao.listWorkInProgressFor(nodeId);
1810+
for (VmWorkJobVO work : works) {
1811+
s_logger.info("Handling unfinished work item: " + work);
1812+
try {
1813+
VMInstanceVO vm = _vmDao.findById(work.getInstanceId());
1814+
if (vm != null) {
1815+
if (work.getType() == State.Starting) {
1816+
_haMgr.scheduleRestart(vm, true);
1817+
work.setManagementServerId(_nodeId);
1818+
_workDao.update(work.getId(), work);
1819+
} else if (work.getType() == State.Stopping) {
1820+
_haMgr.scheduleStop(vm, vm.getHostId(), WorkType.CheckStop);
1821+
work.setManagementServerId(_nodeId);
1822+
_workDao.update(work.getId(), work);
1823+
} else if (work.getType() == State.Migrating) {
1824+
_haMgr.scheduleMigration(vm);
1825+
work.setStep(Step.Done);
1826+
_workDao.update(work.getId(), work);
18281827
}
1829-
} catch (Exception e) {
1830-
s_logger.error("Error while handling " + work, e);
18311828
}
1829+
} catch (Exception e) {
1830+
s_logger.error("Error while handling " + work, e);
18321831
}
1833-
} finally {
1834-
scanLock.unlock();
18351832
}
1833+
} finally {
1834+
scanLock.unlock();
18361835
}
1837-
} finally {
1838-
scanLock.releaseRef();
18391836
}
1837+
} finally {
1838+
scanLock.releaseRef();
1839+
}
18401840
*/
18411841
}
18421842

@@ -3475,7 +3475,7 @@ private void handlePowerOnReportWithNoPendingJobsOnVM(VMInstanceVO vm) {
34753475
}
34763476
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
34773477
VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Running) from out-of-context transition. VM network environment may need to be reset");
3478-
break;
3478+
break;
34793479

34803480
case Destroyed :
34813481
case Expunging :
@@ -3501,30 +3501,29 @@ private void handlePowerOnReportWithNoPendingJobsOnVM(VMInstanceVO vm) {
35013501

35023502
private void handlePowerOffReportWithNoPendingJobsOnVM(VMInstanceVO vm) {
35033503

3504-
// TODO :
35053504
// 1) handle left-over transitional VM states
35063505
// 2) handle out of sync stationary states, schedule force-stop to release resources
35073506
//
35083507
switch(vm.getState()) {
35093508
case Starting :
3510-
break;
3511-
3512-
case Running :
3513-
break;
3514-
35153509
case Stopping :
3516-
break;
3517-
35183510
case Stopped :
3519-
break;
3511+
case Migrating :
3512+
try {
3513+
stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, vm.getPowerHostId());
3514+
} catch(NoTransitionException e) {
3515+
s_logger.warn("Unexpected VM state transition exception, race-condition?", e);
3516+
}
3517+
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
3518+
VM_SYNC_ALERT_SUBJECT, "VM " + vm.getHostName() + "(" + vm.getInstanceName() + ") state is sync-ed (" + vm.getState() + " -> Stopped) from out-of-context transition.");
3519+
// TODO: we need to forcely release all resource allocation
3520+
break;
35203521

3522+
case Running :
35213523
case Destroyed :
35223524
case Expunging :
35233525
break;
35243526

3525-
case Migrating :
3526-
break;
3527-
35283527
case Error :
35293528
default :
35303529
break;
@@ -3582,79 +3581,99 @@ private void scanStalledVMInTransitionStateOnDisconnectedHosts() {
35823581

35833582

35843583
// VMs that in transitional state without recent power state report
3585-
@DB
35863584
private List<Long> listStalledVMInTransitionStateOnUpHost(long hostId, Date cutTime) {
35873585
String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
35883586
"AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " +
35893587
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
3590-
"AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job)";
3588+
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)";
35913589

35923590
List<Long> l = new ArrayList<Long>();
3593-
Transaction txn = Transaction.currentTxn();;
3594-
PreparedStatement pstmt = null;
3595-
try {
3596-
pstmt = txn.prepareAutoCloseStatement(sql);
3597-
3598-
pstmt.setLong(1, hostId);
3599-
pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3600-
ResultSet rs = pstmt.executeQuery();
3601-
while(rs.next()) {
3602-
l.add(rs.getLong(1));
3603-
}
3604-
} catch (SQLException e) {
3605-
} catch (Throwable e) {
3606-
}
3591+
Transaction txn = null;
3592+
try {
3593+
txn = Transaction.open(Transaction.CLOUD_DB);
3594+
3595+
PreparedStatement pstmt = null;
3596+
try {
3597+
pstmt = txn.prepareAutoCloseStatement(sql);
3598+
3599+
pstmt.setLong(1, hostId);
3600+
pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3601+
pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal());
3602+
ResultSet rs = pstmt.executeQuery();
3603+
while(rs.next()) {
3604+
l.add(rs.getLong(1));
3605+
}
3606+
} catch (SQLException e) {
3607+
} catch (Throwable e) {
3608+
}
3609+
3610+
} finally {
3611+
if(txn != null)
3612+
txn.close();
3613+
}
36073614
return l;
36083615
}
36093616

36103617
// VMs that in transitional state and recently have power state update
3611-
@DB
36123618
private List<Long> listVMInTransitionStateWithRecentReportOnUpHost(long hostId, Date cutTime) {
36133619
String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
36143620
"AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " +
36153621
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
3616-
"AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job)";
3622+
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)";
36173623

36183624
List<Long> l = new ArrayList<Long>();
3619-
Transaction txn = Transaction.currentTxn();;
3620-
PreparedStatement pstmt = null;
3621-
try {
3622-
pstmt = txn.prepareAutoCloseStatement(sql);
3623-
3624-
pstmt.setLong(1, hostId);
3625-
pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3626-
ResultSet rs = pstmt.executeQuery();
3627-
while(rs.next()) {
3628-
l.add(rs.getLong(1));
3629-
}
3630-
} catch (SQLException e) {
3631-
} catch (Throwable e) {
3632-
}
3633-
return l;
3625+
Transaction txn = null;
3626+
try {
3627+
txn = Transaction.open(Transaction.CLOUD_DB);
3628+
PreparedStatement pstmt = null;
3629+
try {
3630+
pstmt = txn.prepareAutoCloseStatement(sql);
3631+
3632+
pstmt.setLong(1, hostId);
3633+
pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3634+
pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal());
3635+
ResultSet rs = pstmt.executeQuery();
3636+
while(rs.next()) {
3637+
l.add(rs.getLong(1));
3638+
}
3639+
} catch (SQLException e) {
3640+
} catch (Throwable e) {
3641+
}
3642+
return l;
3643+
} finally {
3644+
if(txn != null)
3645+
txn.close();
3646+
}
36343647
}
36353648

3636-
@DB
36373649
private List<Long> listStalledVMInTransitionStateOnDisconnectedHosts(Date cutTime) {
36383650
String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status != 'UP' " +
36393651
"AND i.power_state_update_time < ? AND i.host_id = h.id " +
36403652
"AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
3641-
"AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job)";
3653+
"AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)";
36423654

36433655
List<Long> l = new ArrayList<Long>();
3644-
Transaction txn = Transaction.currentTxn();;
3645-
PreparedStatement pstmt = null;
3656+
Transaction txn = null;
36463657
try {
3647-
pstmt = txn.prepareAutoCloseStatement(sql);
3648-
3649-
pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3650-
ResultSet rs = pstmt.executeQuery();
3651-
while(rs.next()) {
3652-
l.add(rs.getLong(1));
3653-
}
3654-
} catch (SQLException e) {
3655-
} catch (Throwable e) {
3658+
txn = Transaction.open(Transaction.CLOUD_DB);
3659+
PreparedStatement pstmt = null;
3660+
try {
3661+
pstmt = txn.prepareAutoCloseStatement(sql);
3662+
3663+
pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime));
3664+
pstmt.setInt(2, JobInfo.Status.IN_PROGRESS.ordinal());
3665+
ResultSet rs = pstmt.executeQuery();
3666+
while(rs.next()) {
3667+
l.add(rs.getLong(1));
3668+
}
3669+
} catch (SQLException e) {
3670+
} catch (Throwable e) {
3671+
}
3672+
return l;
3673+
} finally {
3674+
if(txn != null)
3675+
txn.close();
36563676
}
3657-
return l;
36583677
}
36593678

36603679
@Override

‎engine/schema/src/org/apache/cloudstack/vm/jobs/VmWorkJobDaoImpl.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,14 @@ public VmWorkJobDaoImpl() {
4444
@PostConstruct
4545
public void init() {
4646
PendingWorkJobSearch = createSearchBuilder();
47+
PendingWorkJobSearch.and("jobStatus", PendingWorkJobSearch.entity().getStatus(), Op.EQ);
4748
PendingWorkJobSearch.and("vmType", PendingWorkJobSearch.entity().getVmType(), Op.EQ);
4849
PendingWorkJobSearch.and("vmInstanceId", PendingWorkJobSearch.entity().getVmInstanceId(), Op.EQ);
4950
PendingWorkJobSearch.and("step", PendingWorkJobSearch.entity().getStep(), Op.NEQ);
5051
PendingWorkJobSearch.done();
5152

5253
PendingWorkJobByCommandSearch = createSearchBuilder();
54+
PendingWorkJobByCommandSearch.and("jobStatus", PendingWorkJobByCommandSearch.entity().getStatus(), Op.EQ);
5355
PendingWorkJobByCommandSearch.and("vmType", PendingWorkJobByCommandSearch.entity().getVmType(), Op.EQ);
5456
PendingWorkJobByCommandSearch.and("vmInstanceId", PendingWorkJobByCommandSearch.entity().getVmInstanceId(), Op.EQ);
5557
PendingWorkJobByCommandSearch.and("step", PendingWorkJobByCommandSearch.entity().getStep(), Op.NEQ);
@@ -58,17 +60,17 @@ public void init() {
5860

5961
ExpungeWorkJobSearch = createSearchBuilder();
6062
ExpungeWorkJobSearch.and("lastUpdated", ExpungeWorkJobSearch.entity().getLastUpdated(), Op.LT);
61-
ExpungeWorkJobSearch.and("status", ExpungeWorkJobSearch.entity().getStatus(), Op.NEQ);
63+
ExpungeWorkJobSearch.and("jobStatus", ExpungeWorkJobSearch.entity().getStatus(), Op.NEQ);
6264
ExpungeWorkJobSearch.done();
6365
}
6466

6567
@Override
6668
public VmWorkJobVO findPendingWorkJob(VirtualMachine.Type type, long instanceId) {
6769

6870
SearchCriteria<VmWorkJobVO> sc = PendingWorkJobSearch.create();
71+
sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
6972
sc.setParameters("vmType", type);
7073
sc.setParameters("vmInstanceId", instanceId);
71-
sc.setParameters("step", Step.Done);
7274

7375
Filter filter = new Filter(VmWorkJobVO.class, "created", true, null, null);
7476
List<VmWorkJobVO> result = this.listBy(sc, filter);
@@ -82,9 +84,9 @@ public VmWorkJobVO findPendingWorkJob(VirtualMachine.Type type, long instanceId)
8284
public List<VmWorkJobVO> listPendingWorkJobs(VirtualMachine.Type type, long instanceId) {
8385

8486
SearchCriteria<VmWorkJobVO> sc = PendingWorkJobSearch.create();
87+
sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
8588
sc.setParameters("vmType", type);
8689
sc.setParameters("vmInstanceId", instanceId);
87-
sc.setParameters("step", Step.Done);
8890

8991
Filter filter = new Filter(VmWorkJobVO.class, "created", true, null, null);
9092
return this.listBy(sc, filter);
@@ -94,9 +96,9 @@ public List<VmWorkJobVO> listPendingWorkJobs(VirtualMachine.Type type, long inst
9496
public List<VmWorkJobVO> listPendingWorkJobs(VirtualMachine.Type type, long instanceId, String jobCmd) {
9597

9698
SearchCriteria<VmWorkJobVO> sc = PendingWorkJobByCommandSearch.create();
99+
sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
97100
sc.setParameters("vmType", type);
98101
sc.setParameters("vmInstanceId", instanceId);
99-
sc.setParameters("step", Step.Done);
100102
sc.setParameters("cmd", jobCmd);
101103

102104
Filter filter = new Filter(VmWorkJobVO.class, "created", true, null, null);
@@ -115,7 +117,7 @@ public void updateStep(long workJobId, Step step) {
115117
public void expungeCompletedWorkJobs(Date cutDate) {
116118
SearchCriteria<VmWorkJobVO> sc = ExpungeWorkJobSearch.create();
117119
sc.setParameters("lastUpdated",cutDate);
118-
sc.setParameters("status", JobInfo.Status.IN_PROGRESS);
120+
sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
119121

120122
expunge(sc);
121123
}

‎framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ public List<AsyncJobVO> getExpiredCompletedJobs(Date cutTime, int limit) {
180180
public void resetJobProcess(long msid, int jobResultCode, String jobResultMessage) {
181181
String sql = "UPDATE async_job SET job_status=" + JobInfo.Status.FAILED.ordinal() + ", job_result_code=" + jobResultCode
182182
+ ", job_result='" + jobResultMessage + "' where job_status=" + JobInfo.Status.IN_PROGRESS.ordinal()
183-
+ " AND (job_complete_msid=? OR (job_complete_msid IS NULL AND job_init_msid=?))";
183+
+ " AND (job_executing_msid=? OR (job_executing_msid IS NULL AND job_init_msid=?))";
184184

185185
Transaction txn = Transaction.currentTxn();
186186
PreparedStatement pstmt = null;

0 commit comments

Comments
 (0)