Skip to content

Commit 7cdb67d

Browse files
minchen07minchen07
committed
CLOUDSTACK-7589: VM not Starting and always stuck in Stopped state after
management server restarts.
1 parent dd4f6bc commit 7cdb67d

3 files changed

Lines changed: 32 additions & 40 deletions

File tree

‎framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java‎

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -835,24 +835,6 @@ private long getMsid() {
835835
return ManagementServerNode.getManagementServerId();
836836
}
837837

838-
private void cleanupPendingJobs(List<SyncQueueItemVO> l) {
839-
for (SyncQueueItemVO item : l) {
840-
if (s_logger.isInfoEnabled()) {
841-
s_logger.info("Discard left-over queue item: " + item.toString());
842-
}
843-
844-
String contentType = item.getContentType();
845-
if (contentType != null && contentType.equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) {
846-
Long jobId = item.getContentId();
847-
if (jobId != null) {
848-
s_logger.warn("Mark job as failed as its correspoding queue-item has been discarded. job id: " + jobId);
849-
completeAsyncJob(jobId, JobInfo.Status.FAILED, 0, "Execution was cancelled because of server shutdown");
850-
}
851-
}
852-
_queueMgr.purgeItem(item.getId());
853-
}
854-
}
855-
856838
@DB
857839
protected List<Long> wakeupByJoinedJobCompletion(long joinedJobId) {
858840
SearchCriteria<Long> joinJobSC = JoinJobSearch.create("joinJobId", joinedJobId);
@@ -967,25 +949,30 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
967949
return true;
968950
}
969951

952+
private void cleanupLeftOverJobs(final long msid) {
953+
try {
954+
Transaction.execute(new TransactionCallbackNoReturn() {
955+
@Override
956+
public void doInTransactionWithoutResult(TransactionStatus status) {
957+
// purge sync queue item running on this ms node
958+
_queueMgr.cleanupActiveQueueItems(msid, true);
959+
// reset job status for all jobs running on this ms node
960+
_jobDao.resetJobProcess(msid, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart or shutdown");
961+
}
962+
});
963+
} catch (Throwable e) {
964+
s_logger.warn("Unexpected exception in cleaning up left over jobs for mamagement server node " + msid, e);
965+
}
966+
}
967+
970968
@Override
971969
public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
972970
}
973971

974972
@Override
975973
public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
976974
for (final ManagementServerHost msHost : nodeList) {
977-
try {
978-
Transaction.execute(new TransactionCallbackNoReturn() {
979-
@Override
980-
public void doInTransactionWithoutResult(TransactionStatus status) {
981-
List<SyncQueueItemVO> items = _queueMgr.getActiveQueueItems(msHost.getId(), true);
982-
cleanupPendingJobs(items);
983-
_jobDao.resetJobProcess(msHost.getId(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart");
984-
}
985-
});
986-
} catch (Throwable e) {
987-
s_logger.warn("Unexpected exception ", e);
988-
}
975+
cleanupLeftOverJobs(msHost.getId());
989976
}
990977
}
991978

@@ -995,15 +982,7 @@ public void onManagementNodeIsolated() {
995982

996983
@Override
997984
public boolean start() {
998-
try {
999-
_jobDao.cleanupPseduoJobs(getMsid());
1000-
1001-
List<SyncQueueItemVO> l = _queueMgr.getActiveQueueItems(getMsid(), false);
1002-
cleanupPendingJobs(l);
1003-
_jobDao.resetJobProcess(getMsid(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart");
1004-
} catch (Throwable e) {
1005-
s_logger.error("Unexpected exception " + e.getMessage(), e);
1006-
}
985+
cleanupLeftOverJobs(getMsid());
1007986

1008987
_heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS);
1009988
_heartbeatScheduler.scheduleAtFixedRate(getGCTask(), GC_INTERVAL, GC_INTERVAL, TimeUnit.MILLISECONDS);

‎framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,6 @@ public interface SyncQueueManager extends Manager {
3636
public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive);
3737

3838
void purgeAsyncJobQueueItemId(long asyncJobId);
39+
40+
public void cleanupActiveQueueItems(Long msid, boolean exclusive);
3941
}

‎framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626

2727
import org.apache.cloudstack.framework.jobs.dao.SyncQueueDao;
2828
import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao;
29-
3029
import com.cloud.utils.DateUtil;
3130
import com.cloud.utils.component.ManagerBase;
3231
import com.cloud.utils.db.DB;
@@ -260,4 +259,16 @@ public void purgeAsyncJobQueueItemId(long asyncJobId) {
260259
purgeItem(itemId);
261260
}
262261
}
262+
263+
@Override
264+
public void cleanupActiveQueueItems(Long msid, boolean exclusive) {
265+
List<SyncQueueItemVO> l = getActiveQueueItems(msid, false);
266+
for (SyncQueueItemVO item : l) {
267+
if (s_logger.isInfoEnabled()) {
268+
s_logger.info("Discard left-over queue item: " + item.toString());
269+
}
270+
purgeItem(item.getId());
271+
}
272+
}
273+
263274
}

0 commit comments

Comments
 (0)