Skip to content

Commit 59c7150

Browse files
author
abhi
committed
bug 13439: full sync at management server restart, disabling hourly sync
1 parent b0aaf75 commit 59c7150

9 files changed

Lines changed: 71 additions & 100 deletions

File tree

‎api/src/com/cloud/agent/api/ClusterSyncAnswer.java‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,7 @@
2525
public class ClusterSyncAnswer extends Answer {
2626
private long _clusterId;
2727
private HashMap<String, Pair<String, State>> _newStates;
28-
private HashMap<String, Pair<String, State>> _allStates;
29-
private int _type = -1; // 0 for full, 1 for delta
3028
private boolean _isExecuted=false;
31-
32-
public static final int FULL_SYNC=0;
33-
public static final int DELTA_SYNC=1;
3429

3530
// this is here because a cron command answer is being sent twice
3631
// AgentAttache.processAnswers
@@ -47,19 +42,9 @@ public void setExecuted(){
4742
public ClusterSyncAnswer(long clusterId, HashMap<String, Pair<String, State>> newStates){
4843
_clusterId = clusterId;
4944
_newStates = newStates;
50-
_allStates = null;
51-
_type = DELTA_SYNC;
5245
result = true;
5346
}
5447

55-
public ClusterSyncAnswer(long clusterId, HashMap<String, Pair<String, State>> newStates, HashMap<String, Pair<String, State>> allStates){
56-
_clusterId = clusterId;
57-
_newStates = newStates;
58-
_allStates = allStates;
59-
_type = FULL_SYNC;
60-
result = true;
61-
}
62-
6348
public long getClusterId() {
6449
return _clusterId;
6550
}
@@ -68,15 +53,4 @@ public HashMap<String, Pair<String, State>> getNewStates() {
6853
return _newStates;
6954
}
7055

71-
public HashMap<String, Pair<String, State>> getAllStates() {
72-
return _allStates;
73-
}
74-
75-
public boolean isFull(){
76-
return _type==0;
77-
}
78-
79-
public boolean isDelta(){
80-
return _type==1;
81-
}
8256
}

‎api/src/com/cloud/agent/api/ClusterSyncCommand.java‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,39 +20,22 @@
2020

2121
public class ClusterSyncCommand extends Command implements CronCommand {
2222
int _interval;
23-
int _skipSteps; // skip this many steps for full sync
24-
int _steps;
2523

2624
long _clusterId;
2725

2826
public ClusterSyncCommand() {
2927
}
3028

31-
public ClusterSyncCommand(int interval, int skipSteps, long clusterId){
29+
public ClusterSyncCommand(int interval, long clusterId){
3230
_interval = interval;
33-
_skipSteps = skipSteps;
3431
_clusterId = clusterId;
35-
_steps=0;
3632
}
3733

3834
@Override
3935
public int getInterval() {
4036
return _interval;
4137
}
4238

43-
public int getSkipSteps(){
44-
return _skipSteps;
45-
}
46-
47-
public void incrStep(){
48-
_steps++;
49-
if (_steps>=_skipSteps)_steps=0;
50-
}
51-
52-
public boolean isRightStep(){
53-
return (_steps==0);
54-
}
55-
5639
public long getClusterId() {
5740
return _clusterId;
5841
}

‎core/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6626,16 +6626,7 @@ protected Answer execute(final ClusterSyncCommand cmd) {
66266626
return new Answer(cmd);
66276627
}
66286628
HashMap<String, Pair<String, State>> newStates = deltaClusterSync(conn);
6629-
cmd.incrStep();
6630-
if (cmd.isRightStep()){
6631-
// do full sync
6632-
HashMap<String, Pair<String, State>> allStates=fullClusterSync(conn);
6633-
return new ClusterSyncAnswer(cmd.getClusterId(), newStates, allStates);
6634-
}
6635-
else {
6636-
cmd.incrStep();
6637-
return new ClusterSyncAnswer(cmd.getClusterId(), newStates);
6638-
}
6629+
return new ClusterSyncAnswer(cmd.getClusterId(), newStates);
66396630
}
66406631

66416632

‎server/src/com/cloud/configuration/Config.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,6 @@ public enum Config {
160160
PingInterval("Advanced", AgentManager.class, Integer.class, "ping.interval", "60", "Ping interval in seconds", null),
161161
PingTimeout("Advanced", AgentManager.class, Float.class, "ping.timeout", "2.5", "Multiplier to ping.interval before announcing an agent has timed out", null),
162162
ClusterDeltaSyncInterval("Advanced", AgentManager.class, Integer.class, "sync.interval", "60", "Cluster Delta sync interval in seconds", null),
163-
ClusterFullSyncSkipSteps("Advanced", AgentManager.class, Integer.class, "skip.steps", "60", "Cluster full sync skip steps count", null),
164163
Port("Advanced", AgentManager.class, Integer.class, "port", "8250", "Port to listen on for agent connection.", null),
165164
RouterCpuMHz("Advanced", NetworkManager.class, Integer.class, "router.cpu.mhz", String.valueOf(VirtualNetworkApplianceManager.DEFAULT_ROUTER_CPU_MHZ), "Default CPU speed (MHz) for router VM.", null),
166165
RestartRetryInterval("Advanced", HighAvailabilityManager.class, Integer.class, "restart.retry.interval", "600", "Time (in seconds) between retries to restart a vm", null),

‎server/src/com/cloud/host/dao/HostDao.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,5 +66,7 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
6666

6767
List<HostVO> findAndUpdateApplianceToLoad(long lastPingSecondsAfter, long managementServerId);
6868

69-
boolean updateResourceState(ResourceState oldState, ResourceState.Event event, ResourceState newState, Host vo);
69+
boolean updateResourceState(ResourceState oldState, ResourceState.Event event, ResourceState newState, Host vo);
70+
71+
HostVO findByGuid(String guid);
7072
}

‎server/src/com/cloud/host/dao/HostDaoImpl.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,14 @@ public long countBy(long clusterId, ResourceState... states) {
319319
List<HostVO> hosts = listBy(sc);
320320
return hosts.size();
321321
}
322+
323+
324+
@Override
325+
public HostVO findByGuid(String guid) {
326+
SearchCriteria<HostVO> sc = GuidSearch.create("guid", guid);
327+
return findOneBy(sc);
328+
}
329+
322330
@Override @DB
323331
public List<HostVO> findAndUpdateDirectAgentToLoad(long lastPingSecondsAfter, Long limit, long managementServerId) {
324332
Transaction txn = Transaction.currentTxn();

‎server/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1706,43 +1706,62 @@ public void deltaSync(Map<String, Pair<String, State>> newStates) {
17061706
}
17071707

17081708

1709-
public void fullSync(final long clusterId, Map<String, Pair<String, State>> newStates, boolean init) {
1709+
public void fullSync(final long clusterId, Map<String, Pair<String, State>> newStates) {
1710+
if (newStates==null)return;
17101711
Map<Long, AgentVmInfo> infos = convertToInfos(newStates);
17111712
Set<VMInstanceVO> set_vms = Collections.synchronizedSet(new HashSet<VMInstanceVO>());
17121713
set_vms.addAll(_vmDao.listByClusterId(clusterId));
1713-
set_vms.addAll(_vmDao.listStartingByClusterId(clusterId));
1714-
1714+
set_vms.addAll(_vmDao.listLHByClusterId(clusterId));
1715+
17151716
for (VMInstanceVO vm : set_vms) {
17161717
if (vm.isRemoved() || vm.getState() == State.Destroyed || vm.getState() == State.Expunging) continue;
17171718
AgentVmInfo info = infos.remove(vm.getId());
1718-
if (init){ // mark the VMs real state on initial sync
1719-
VMInstanceVO castedVm = null;
1720-
if (info == null && vm.getState() == State.Running) { // only work on VMs which were supposed to be running earlier
1721-
info = new AgentVmInfo(vm.getInstanceName(), getVmGuru(vm), vm, State.Stopped);
1722-
castedVm = info.guru.findById(vm.getId());
1723-
try {
1724-
Host host = _resourceMgr.findHostByGuid(info.getHostUuid());
1725-
long hostId = host == null ? (vm.getHostId() == null ? vm.getLastHostId() : vm.getHostId()) : host.getId();
1726-
HypervisorGuru hvGuru = _hvGuruMgr.getGuru(castedVm.getHypervisorType());
1727-
Command command = compareState(hostId, castedVm, info, true, hvGuru.trackVmHostChange());
1728-
if (command != null){
1729-
Answer answer = _agentMgr.send(hostId, command);
1730-
if (!answer.getResult()) {
1731-
s_logger.warn("Failed to update state of the VM due to " + answer.getDetails());
1732-
}
1733-
}
1734-
} catch (Exception e) {
1735-
s_logger.warn("Unable to update state of the VM due to exception " + e.getMessage());
1736-
e.printStackTrace();
1737-
}
1719+
VMInstanceVO castedVm = null;
1720+
if ((info == null && (vm.getState() == State.Running || vm.getState() == State.Starting))
1721+
|| (info != null && (info.state == State.Running && vm.getState() == State.Starting)))
1722+
{
1723+
s_logger.info("Found vm " + vm.getInstanceName() + " in inconsistent state. " + vm.getState() + " on CS while " + (info == null ? "Stopped" : "Running") + " on agent");
1724+
info = new AgentVmInfo(vm.getInstanceName(), getVmGuru(vm), vm, State.Stopped);
1725+
vm.setState(State.Running); // set it as running and let HA take care of it
1726+
_vmDao.persist(vm);
1727+
castedVm = info.guru.findById(vm.getId());
1728+
try {
1729+
Host host = _hostDao.findByGuid(info.getHostUuid());
1730+
long hostId = host == null ? (vm.getHostId() == null ? vm.getLastHostId() : vm.getHostId()) : host.getId();
1731+
HypervisorGuru hvGuru = _hvGuruMgr.getGuru(castedVm.getHypervisorType());
1732+
Command command = compareState(hostId, castedVm, info, true, hvGuru.trackVmHostChange());
1733+
if (command != null){
1734+
Answer answer = _agentMgr.send(hostId, command);
1735+
if (!answer.getResult()) {
1736+
s_logger.warn("Failed to update state of the VM due to " + answer.getDetails());
1737+
}
1738+
}
1739+
} catch (Exception e) {
1740+
s_logger.warn("Unable to update state of the VM due to exception " + e.getMessage());
1741+
e.printStackTrace();
17381742
}
1739-
}
1740-
1741-
}
1743+
}
1744+
else
1745+
// host id can change
1746+
if (info != null && vm.getState() == State.Running){
1747+
// check for host id changes
1748+
Host host = _hostDao.findByGuid(info.getHostUuid());
1749+
if (host != null && (vm.getHostId() == null || host.getId() != vm.getHostId())){
1750+
s_logger.info("Found vm " + vm.getInstanceName() + " with inconsistent host in db, new host is " + host.getId());
1751+
try {
1752+
stateTransitTo(vm, VirtualMachine.Event.AgentReportMigrated, host.getId());
1753+
} catch (NoTransitionException e) {
1754+
s_logger.warn(e.getMessage());
1755+
}
1756+
}
1757+
}
17421758

1759+
}
1760+
17431761
for (final AgentVmInfo left : infos.values()) {
1762+
if (VirtualMachineName.isValidVmName(left.name)) continue; // if the vm follows cloudstack naming ignore it for stopping
17441763
try {
1745-
Host host = _resourceMgr.findHostByGuid(left.getHostUuid());
1764+
Host host = _hostDao.findByGuid(left.getHostUuid());
17461765
if (host != null){
17471766
s_logger.warn("Stopping a VM which we do not have any record of " + left.name);
17481767
Answer answer = _agentMgr.send(host.getId(), cleanup(left.name));
@@ -1754,10 +1773,11 @@ public void fullSync(final long clusterId, Map<String, Pair<String, State>> newS
17541773
s_logger.warn("Unable to stop a VM due to " + e.getMessage());
17551774
}
17561775
}
1757-
1776+
17581777
}
17591778

17601779

1780+
17611781
protected Map<Long, AgentVmInfo> convertToInfos(final Map<String, Pair<String, State>> newStates) {
17621782
final HashMap<Long, AgentVmInfo> map = new HashMap<Long, AgentVmInfo>();
17631783
if (newStates == null) {
@@ -2088,12 +2108,7 @@ public boolean processAnswers(long agentId, long seq, Answer[] answers) {
20882108
if (answer instanceof ClusterSyncAnswer) {
20892109
ClusterSyncAnswer hs = (ClusterSyncAnswer) answer;
20902110
if (!hs.isExceuted()){
2091-
if (hs.isFull()) {
2092-
deltaSync(hs.getNewStates());
2093-
fullSync(hs.getClusterId(), hs.getAllStates(), false);
2094-
} else if (hs.isDelta()){
2095-
deltaSync(hs.getNewStates());
2096-
}
2111+
deltaSync(hs.getNewStates());
20972112
hs.setExecuted();
20982113
}
20992114
} else if (!answer.getResult()) {
@@ -2171,12 +2186,11 @@ public void processConnect(HostVO agent, StartupCommand cmd, boolean forRebalanc
21712186
StartupRoutingCommand startup = (StartupRoutingCommand) cmd;
21722187
HashMap<String, Pair<String, State>> allStates = startup.getClusterVMStateChanges();
21732188
if (allStates != null){
2174-
this.fullSync(clusterId, allStates, true);
2189+
this.fullSync(clusterId, allStates);
21752190
}
21762191

21772192
// initiate the cron job
2178-
ClusterSyncCommand syncCmd = new ClusterSyncCommand(Integer.parseInt(Config.ClusterDeltaSyncInterval.getDefaultValue()),
2179-
Integer.parseInt(Config.ClusterFullSyncSkipSteps.getDefaultValue()), clusterId);
2193+
ClusterSyncCommand syncCmd = new ClusterSyncCommand(Integer.parseInt(Config.ClusterDeltaSyncInterval.getDefaultValue()), clusterId);
21802194
try {
21812195
long seq_no = _agentMgr.send(agentId, new Commands(syncCmd), this);
21822196
s_logger.debug("Cluster VM sync started with jobid " + seq_no);

‎server/src/com/cloud/vm/dao/VMInstanceDao.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
8484
public Long countAllocatedVirtualRoutersForAccount(long accountId);
8585

8686
List<VMInstanceVO> listByClusterId(long clusterId); // this does not pull up VMs which are starting
87-
List<VMInstanceVO> listStartingByClusterId(long clusterId); // get all the VMs even starting one on this cluster
87+
List<VMInstanceVO> listLHByClusterId(long clusterId); // get all the VMs even starting one on this cluster
8888

8989
List<VMInstanceVO> listVmsMigratingFromHost(Long hostId);
9090

‎server/src/com/cloud/vm/dao/VMInstanceDaoImpl.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
5959
public static final Logger s_logger = Logger.getLogger(VMInstanceDaoImpl.class);
6060

6161
protected final SearchBuilder<VMInstanceVO> VMClusterSearch;
62-
protected final SearchBuilder<VMInstanceVO> StartingVMClusterSearch;
62+
protected final SearchBuilder<VMInstanceVO> LHVMClusterSearch;
6363
protected final SearchBuilder<VMInstanceVO> IdStatesSearch;
6464
protected final SearchBuilder<VMInstanceVO> AllFieldsSearch;
6565
protected final SearchBuilder<VMInstanceVO> ZoneTemplateNonExpungedSearch;
@@ -102,11 +102,11 @@ protected VMInstanceDaoImpl() {
102102
VMClusterSearch.done();
103103

104104

105-
StartingVMClusterSearch = createSearchBuilder();
105+
LHVMClusterSearch = createSearchBuilder();
106106
SearchBuilder<HostVO> hostSearch1 = _hostDao.createSearchBuilder();
107-
StartingVMClusterSearch.join("hostSearch1", hostSearch1, hostSearch1.entity().getId(), StartingVMClusterSearch.entity().getHostId(), JoinType.INNER);
107+
LHVMClusterSearch.join("hostSearch1", hostSearch1, hostSearch1.entity().getId(), LHVMClusterSearch.entity().getLastHostId(), JoinType.INNER);
108108
hostSearch1.and("clusterId", hostSearch1.entity().getClusterId(), SearchCriteria.Op.EQ);
109-
StartingVMClusterSearch.done();
109+
LHVMClusterSearch.done();
110110

111111

112112
AllFieldsSearch = createSearchBuilder();
@@ -227,8 +227,8 @@ public List<VMInstanceVO> listByClusterId(long clusterId) {
227227

228228

229229
@Override
230-
public List<VMInstanceVO> listStartingByClusterId(long clusterId) {
231-
SearchCriteria<VMInstanceVO> sc = StartingVMClusterSearch.create();
230+
public List<VMInstanceVO> listLHByClusterId(long clusterId) {
231+
SearchCriteria<VMInstanceVO> sc = LHVMClusterSearch.create();
232232
sc.setJoinParameters("hostSearch1", "clusterId", clusterId);
233233
return listBy(sc);
234234
}

0 commit comments

Comments
 (0)