Skip to content

Commit b0f0efa

Browse files
author
Alex Huang
committed
more changes to make it work better in a cluster
1 parent db7bc89 commit b0f0efa

5 files changed

Lines changed: 92 additions & 113 deletions

File tree

‎api/src/com/cloud/vm/VirtualMachine.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,6 @@ public static StateMachine2<State, VirtualMachine.Event, VirtualMachine> getStat
8888
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.MigrationRequested, State.Migrating);
8989
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.OperationSucceeded, State.Running);
9090
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.OperationFailed, State.Running);
91-
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.MigrationFailedOnSource, State.Running);
92-
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.MigrationFailedOnDest, State.Running);
9391
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.AgentReportRunning, State.Running);
9492
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.AgentReportStopped, State.Stopped);
9593
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.AgentReportShutdowned, State.Stopped);
@@ -139,10 +137,7 @@ public enum Event {
139137
ExpungeOperation,
140138
OperationSucceeded,
141139
OperationFailed,
142-
MigrationFailedOnSource,
143-
MigrationFailedOnDest,
144140
OperationRetry,
145-
OperationCancelled,
146141
AgentReportShutdowned
147142
};
148143

‎server/src/com/cloud/agent/manager/AgentManagerImpl.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1206,6 +1206,12 @@ public Answer send(Long hostId, Command cmd, int timeout)
12061206

12071207
return null;
12081208
}
1209+
1210+
@DB
1211+
protected boolean noDbTxn() {
1212+
Transaction txn = Transaction.currentTxn();
1213+
return !txn.dbTxnStarted();
1214+
}
12091215

12101216
@Override
12111217
public Answer[] send(Long hostId, Commands commands, int timeout)
@@ -1214,6 +1220,8 @@ public Answer[] send(Long hostId, Commands commands, int timeout)
12141220
if (hostId == null) {
12151221
throw new AgentUnavailableException(-1);
12161222
}
1223+
1224+
assert noDbTxn() : "I know, I know. Why are we so strict as to not allow txn across an agent call? ... Why are we so cruel ... Why are we such a dictator .... Too bad... Sorry...but NO AGENT COMMANDS WRAPPED WITHIN DB TRANSACTIONS!";
12171225

12181226
Command[] cmds = commands.toCommands();
12191227

‎server/src/com/cloud/vm/ItWorkVO.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
import javax.persistence.Column;
2121
import javax.persistence.Entity;
22+
import javax.persistence.EnumType;
23+
import javax.persistence.Enumerated;
2224
import javax.persistence.Id;
2325
import javax.persistence.Table;
2426

@@ -30,15 +32,17 @@
3032
public class ItWorkVO {
3133
enum ResourceType {
3234
Volume,
33-
Nic
35+
Nic,
36+
Host
3437
}
3538

3639
enum Step {
3740
Prepare,
38-
Start,
41+
Starting,
3942
Started,
4043
Release,
41-
Done
44+
Done,
45+
Migrating
4246
}
4347

4448
@Id
@@ -76,6 +80,13 @@ public long getInstanceId() {
7680
@Column(name="resource_type")
7781
ResourceType resourceType;
7882

83+
@Column(name="vm_type")
84+
@Enumerated(value=EnumType.STRING)
85+
VirtualMachine.Type vmType;
86+
87+
public VirtualMachine.Type getVmType() {
88+
return vmType;
89+
}
7990

8091
public long getResourceId() {
8192
return resourceId;
@@ -96,7 +107,7 @@ public void setResourceType(ResourceType resourceType) {
96107
protected ItWorkVO() {
97108
}
98109

99-
protected ItWorkVO(String id, long managementServerId, State type, long instanceId) {
110+
protected ItWorkVO(String id, long managementServerId, State type, VirtualMachine.Type vmType, long instanceId) {
100111
this.id = id;
101112
this.managementServerId = managementServerId;
102113
this.type = type;
@@ -106,6 +117,7 @@ protected ItWorkVO(String id, long managementServerId, State type, long instance
106117
this.resourceType = null;
107118
this.createdAt = InaccurateClock.getTimeInSeconds();
108119
this.updatedAt = createdAt;
120+
this.vmType = vmType;
109121
}
110122

111123
public String getId() {

‎server/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 66 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@
125125
import com.cloud.utils.db.GlobalLock;
126126
import com.cloud.utils.db.Transaction;
127127
import com.cloud.utils.exception.CloudRuntimeException;
128-
import com.cloud.utils.fsm.StateListener;
129128
import com.cloud.utils.fsm.StateMachine2;
130129
import com.cloud.vm.ItWorkVO.Step;
131130
import com.cloud.vm.VirtualMachine.Event;
@@ -261,60 +260,6 @@ public <T extends VMInstanceVO> T allocate(T vm,
261260
return vm;
262261
}
263262

264-
265-
266-
protected void reserveNics(VirtualMachineProfile<? extends VMInstanceVO> vmProfile, DeployDestination dest, ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {
267-
// List<NicVO> nics = _nicsDao.listBy(vmProfile.getId());
268-
// for (NicVO nic : nics) {
269-
// Pair<NetworkGuru, NetworkVO> implemented = _networkMgr.implementNetwork(nic.getNetworkId(), dest, context);
270-
// NetworkGuru concierge = implemented.first();
271-
// NetworkVO network = implemented.second();
272-
// NicProfile profile = null;
273-
// if (nic.getReservationStrategy() == ReservationStrategy.Start) {
274-
// nic.setState(Resource.State.Reserving);
275-
// nic.setReservationId(context.getReservationId());
276-
// _nicsDao.update(nic.getId(), nic);
277-
// URI broadcastUri = nic.getBroadcastUri();
278-
// if (broadcastUri == null) {
279-
// network.getBroadcastUri();
280-
// }
281-
//
282-
// URI isolationUri = nic.getIsolationUri();
283-
//
284-
// profile = new NicProfile(nic, network, broadcastUri, isolationUri);
285-
// concierge.reserve(profile, network, vmProfile, dest, context);
286-
// nic.setIp4Address(profile.getIp4Address());
287-
// nic.setIp6Address(profile.getIp6Address());
288-
// nic.setMacAddress(profile.getMacAddress());
289-
// nic.setIsolationUri(profile.getIsolationUri());
290-
// nic.setBroadcastUri(profile.getBroadCastUri());
291-
// nic.setReserver(concierge.getName());
292-
// nic.setState(Resource.State.Reserved);
293-
// nic.setNetmask(profile.getNetmask());
294-
// nic.setGateway(profile.getGateway());
295-
// nic.setAddressFormat(profile.getFormat());
296-
// _nicsDao.update(nic.getId(), nic);
297-
// } else {
298-
// profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri());
299-
// }
300-
//
301-
// for (NetworkElement element : _networkElements) {
302-
// if (s_logger.isDebugEnabled()) {
303-
// s_logger.debug("Asking " + element.getName() + " to prepare for " + nic);
304-
// }
305-
// element.prepare(network, profile, vmProfile, dest, context);
306-
// }
307-
//
308-
// vmProfile.addNic(profile);
309-
// _networksDao.changeActiveNicsBy(network.getId(), 1);
310-
// }
311-
}
312-
313-
protected void prepareNics(VirtualMachineProfile<? extends VMInstanceVO> vmProfile, DeployDestination dest, ReservationContext context) {
314-
315-
}
316-
317-
318263
@Override
319264
public <T extends VMInstanceVO> T allocate(T vm,
320265
VMTemplateVO template,
@@ -353,7 +298,7 @@ public <T extends VMInstanceVO> boolean expunge(T vm, User caller, Account accou
353298
try {
354299
if (advanceExpunge(vm, caller, account)) {
355300
//Mark vms as removed
356-
remove(vm, _accountMgr.getSystemUser(), account);
301+
remove(vm, caller, account);
357302
return true;
358303
} else {
359304
s_logger.info("Did not expunge " + vm);
@@ -507,7 +452,7 @@ protected boolean checkWorkItems(VMInstanceVO vm, State state) throws Concurrent
507452
protected <T extends VMInstanceVO> Ternary<T, ReservationContext, ItWorkVO> changeToStartState(VirtualMachineGuru<T> vmGuru, T vm, User caller, Account account) throws ConcurrentOperationException {
508453
long vmId = vm.getId();
509454

510-
ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Starting, vm.getId());
455+
ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Starting, vm.getType(), vm.getId());
511456
int retry = _lockStateRetry;
512457
while (retry-- != 0) {
513458
Transaction txn = Transaction.currentTxn();
@@ -643,7 +588,7 @@ public <T extends VMInstanceVO> T advanceStart(T vm, Map<VirtualMachineProfile.P
643588
if (work == null || work.getStep() != Step.Prepare) {
644589
throw new ConcurrentOperationException("Work steps have been changed: " + work);
645590
}
646-
_workDao.updateStep(work, Step.Start);
591+
_workDao.updateStep(work, Step.Starting);
647592

648593
_agentMgr.send(destHostId, cmds);
649594
_workDao.updateStep(work, Step.Started);
@@ -749,12 +694,12 @@ protected <T extends VMInstanceVO> boolean cleanup(VirtualMachineGuru<T> guru, V
749694
s_logger.debug("Cleaning up resources for the vm " + vm + " in " + state + " state");
750695
if (state == State.Starting) {
751696
Step step = work.getStep();
752-
if (step == Step.Start && !force) {
697+
if (step == Step.Starting && !force) {
753698
s_logger.warn("Unable to cleanup vm " + vm + "; work state is incorrect: " + step);
754699
return false;
755700
}
756701

757-
if (step == Step.Started || step == Step.Start) {
702+
if (step == Step.Started || step == Step.Starting) {
758703
if (vm.getHostId() != null) {
759704
if (!sendStop(guru, profile, force)) {
760705
s_logger.warn("Failed to stop vm " + vm + " in " + State.Starting + " state as a part of cleanup process");
@@ -763,7 +708,7 @@ protected <T extends VMInstanceVO> boolean cleanup(VirtualMachineGuru<T> guru, V
763708
}
764709
}
765710

766-
if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Start) {
711+
if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Starting) {
767712
s_logger.debug("Cleanup is not needed for vm " + vm + "; work state is incorrect: " + step);
768713
return true;
769714
}
@@ -950,6 +895,14 @@ public <T extends VMInstanceVO> boolean destroy(T vm, User user, Account caller)
950895
return true;
951896
}
952897

898+
protected boolean checkVmOnHost(VirtualMachine vm, long hostId) throws AgentUnavailableException, OperationTimedoutException {
899+
CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer)_agentMgr.send(hostId, new CheckVirtualMachineCommand(vm.getInstanceName()));
900+
if (!answer.getResult() || answer.getState() == State.Stopped) {
901+
return false;
902+
}
903+
904+
return true;
905+
}
953906

954907
@Override
955908
public <T extends VMInstanceVO> T migrate(T vm, long srcHostId, DeployDestination dest) throws ResourceUnavailableException {
@@ -964,7 +917,7 @@ public <T extends VMInstanceVO> T migrate(T vm, long srcHostId, DeployDestinatio
964917
VirtualMachineGuru<T> vmGuru = getVmGuru(vm);
965918

966919
vm = vmGuru.findById(vm.getId());
967-
if (vm == null || vm.getRemoved() != null) {
920+
if (vm == null) {
968921
if (s_logger.isDebugEnabled()) {
969922
s_logger.debug("Unable to find the vm " + vm);
970923
}
@@ -986,69 +939,80 @@ public <T extends VMInstanceVO> T migrate(T vm, long srcHostId, DeployDestinatio
986939
VirtualMachineTO to = hvGuru.implement(profile);
987940
PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to);
988941

989-
PrepareForMigrationAnswer pfma;
942+
ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId());
943+
work.setStep(Step.Prepare);
944+
work.setResourceType(ItWorkVO.ResourceType.Host);
945+
work.setResourceId(dstHostId);
946+
_workDao.persist(work);
947+
948+
PrepareForMigrationAnswer pfma = null;
990949
try {
991950
pfma = (PrepareForMigrationAnswer)_agentMgr.send(dstHostId, pfmc);
951+
if (!pfma.getResult()) {
952+
String msg = "Unable to prepare for migration due to " + pfma.getDetails();
953+
pfma = null;
954+
throw new AgentUnavailableException(msg, dstHostId);
955+
}
992956
} catch (OperationTimedoutException e1) {
993957
throw new AgentUnavailableException("Operation timed out", dstHostId);
958+
} finally {
959+
if (pfma == null) {
960+
work.setStep(Step.Done);
961+
_workDao.update(work.getId(), work);
962+
}
994963
}
995-
if (!pfma.getResult()) {
996-
throw new AgentUnavailableException(pfma.getDetails(), dstHostId);
997-
}
964+
965+
vm.setLastHostId(srcHostId);
966+
if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) {
967+
s_logger.info("Migration cancelled because state has changed: " + vm);
968+
return null;
969+
}
998970

999971
boolean migrated = false;
1000972
try {
1001-
vm.setLastHostId(srcHostId);
1002-
if (vm == null || vm.getRemoved() != null || vm.getHostId() == null || vm.getHostId() != srcHostId || !stateTransitTo(vm, Event.MigrationRequested, dstHostId)) {
1003-
s_logger.info("Migration cancelled because state has changed: " + vm);
1004-
return null;
1005-
}
1006-
1007973
boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
1008974
MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows);
1009-
MigrateAnswer ma = (MigrateAnswer)_agentMgr.send(vm.getLastHostId(), mc);
1010-
if (!ma.getResult()) {
1011-
return null;
975+
try {
976+
MigrateAnswer ma = (MigrateAnswer)_agentMgr.send(vm.getLastHostId(), mc);
977+
if (!ma.getResult()) {
978+
s_logger.error("Unable to migrate due to " + ma.getDetails());
979+
return null;
980+
}
981+
} catch (OperationTimedoutException e) {
982+
if (e.isActive()) {
983+
s_logger.warn("Active migration command so scheduling a restart for " + vm);
984+
_haMgr.scheduleRestart(vm, true);
985+
}
986+
throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId);
1012987
}
1013-
Commands cmds = new Commands(OnError.Revert);
1014-
CheckVirtualMachineCommand cvm = new CheckVirtualMachineCommand(vm.getInstanceName());
1015-
cmds.addCommand(cvm);
1016988

1017-
_agentMgr.send(dstHostId, cmds);
1018-
CheckVirtualMachineAnswer answer = cmds.getAnswer(CheckVirtualMachineAnswer.class);
1019-
if (!answer.getResult()) {
1020-
s_logger.debug("Unable to complete migration for " + vm.toString());
1021-
stateTransitTo(vm, VirtualMachine.Event.AgentReportStopped, null);
1022-
return null;
1023-
}
1024-
1025-
State state = answer.getState();
1026-
if (state == State.Stopped) {
1027-
s_logger.warn("Unable to complete migration as we can not detect it on " + dest.getHost());
1028-
stateTransitTo(vm, VirtualMachine.Event.AgentReportStopped, null);
1029-
return null;
989+
changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started);
990+
991+
try {
992+
if (!checkVmOnHost(vm, dstHostId)) {
993+
s_logger.error("Unable to complete migration for " + vm);
994+
_agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null);
995+
cleanup(vmGuru, new VirtualMachineProfileImpl<T>(vm), work, Event.AgentReportStopped, true, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount());
996+
return null;
997+
}
998+
} catch (OperationTimedoutException e) {
1030999
}
1031-
1032-
stateTransitTo(vm, VirtualMachine.Event.OperationSucceeded, dstHostId);
1000+
10331001
migrated = true;
10341002
return vm;
1035-
} catch (final OperationTimedoutException e) {
1036-
s_logger.debug("operation timed out");
1037-
if (e.isActive()) {
1038-
// FIXME: scheduleRestart(vm, true);
1039-
}
1040-
throw new AgentUnavailableException("Operation timed out: ", dstHostId);
10411003
} finally {
10421004
if (!migrated) {
10431005
s_logger.info("Migration was unsuccessful. Cleaning up: " + vm);
10441006

10451007
_alertMgr.sendAlert(alertType, fromHost.getDataCenterId(), fromHost.getPodId(), "Unable to migrate vm " + vm.getName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " + dest.getPod().getName(), "Migrate Command failed. Please check logs.");
10461008

1047-
stateTransitTo(vm, Event.MigrationFailedOnSource, srcHostId);
1009+
_agentMgr.send(dstHostId, new Commands(cleanup(vm.getInstanceName())), null);
10481010

1049-
Command cleanup = cleanup(vm.getInstanceName());
1050-
_agentMgr.easySend(dstHostId, cleanup);
1011+
stateTransitTo(vm, Event.OperationFailed, srcHostId);
10511012
}
1013+
1014+
work.setStep(Step.Done);
1015+
_workDao.update(work.getId(), work);
10521016
}
10531017
}
10541018

@@ -1199,7 +1163,6 @@ public <T extends VMInstanceVO> T advanceReboot(T vm, Map<VirtualMachineProfile.
11991163
return rebootedVm;
12001164
}
12011165

1202-
12031166
@Override
12041167
public VMInstanceVO findById(VirtualMachine.Type type, long vmId) {
12051168
VirtualMachineGuru<? extends VMInstanceVO> guru = _vmGurus.get(type);

‎setup/db/create-schema.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,12 @@ DROP TABLE IF EXISTS `cloud`.`usage_event`;
9393
DROP TABLE IF EXISTS `cloud`.`host_tags`;
9494

9595
CREATE TABLE `cloud`.`op_it_work` (
96-
`id` char(40) COMMENT 'id',
96+
`id` char(40) COMMENT 'reservation id',
9797
`mgmt_server_id` bigint unsigned COMMENT 'management server id',
9898
`created_at` bigint unsigned NOT NULL COMMENT 'when was this work detail created',
9999
`thread` varchar(255) NOT NULL COMMENT 'thread name',
100100
`type` char(32) NOT NULL COMMENT 'type of work',
101+
`vm_type` char(32) NOT NULL COMMENT 'type of vm',
101102
`step` char(32) NOT NULL COMMENT 'state',
102103
`updated_at` bigint unsigned NOT NULL COMMENT 'time it was taken over',
103104
`instance_id` bigint unsigned NOT NULL COMMENT 'vm instance',

0 commit comments

Comments
 (0)