Skip to content

Commit 9909371

Browse files
author
Alex Huang
committed
bug 9445: Signal alert for the host if a primary storage pool was unavailable on that host
1 parent 4290abb commit 9909371

5 files changed

Lines changed: 92 additions & 66 deletions

File tree

‎api/src/com/cloud/exception/ConnectionException.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,26 @@
1919

2020
import com.cloud.utils.SerialVersionUID;
2121

22+
/**
23+
* ConnectionException is thrown by Listeners while processing the startup
24+
* command. There are two uses for this exception and they are distinguished
25+
* by the boolean flag.
26+
* 1. If the flag is set to true, there is an unexpected error during the
27+
* processing. Upon receiving this exception, the AgentManager will
28+
* immediately place the agent under alert. When the function to enable
29+
* to disable the agent, the agent is disabled.
30+
* 2. If the flag is set to false, the listener has decided that the resource
31+
* should be disconnected and reconnected to "refresh" all resource
32+
* information. This is useful when the Listener needed to perform setup
33+
* on the agent and decided it is best to flush connection and reconnect.
34+
* It is important that the Listener does not fall into a loop in this
35+
* situation where it keeps throwing ConnectionException.
36+
*/
2237
public class ConnectionException extends Exception {
2338

2439
private static final long serialVersionUID = SerialVersionUID.ConnectionException;
2540
boolean _error;
26-
41+
2742
public ConnectionException(boolean setupError, String msg) {
2843
this(setupError, msg, null);
2944
}

‎server/src/com/cloud/agent/manager/AgentManagerImpl.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1602,15 +1602,22 @@ protected AgentAttache notifyMonitorsOfConnection(AgentAttache attache, final St
16021602
for (int i = 0; i < cmd.length; i++) {
16031603
try {
16041604
monitor.second().processConnect(host, cmd[i]);
1605-
} catch (ConnectionException e) {
1606-
if (e.isSetupError()) {
1607-
s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage());
1608-
handleDisconnect(attache, Event.AgentDisconnected, false);
1609-
throw e;
1605+
} catch (Exception e) {
1606+
if (e instanceof ConnectionException) {
1607+
ConnectionException ce = (ConnectionException)e;
1608+
if (ce.isSetupError()) {
1609+
s_logger.warn("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage());
1610+
handleDisconnect(attache, Event.AgentDisconnected, false);
1611+
throw ce;
1612+
} else {
1613+
s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + e.getMessage());
1614+
handleDisconnect(attache, Event.ShutdownRequested, false);
1615+
return attache;
1616+
}
16101617
} else {
1611-
s_logger.info("Monitor " + monitor.second().getClass().getSimpleName() + " says not to continue the connect process for " + hostId + " due to " + e.getMessage());
1612-
handleDisconnect(attache, Event.ShutdownRequested, false);
1613-
return attache;
1618+
s_logger.error("Monitor " + monitor.second().getClass().getSimpleName() + " says there is an error in the connect process for " + hostId + " due to " + e.getMessage(), e);
1619+
handleDisconnect(attache, Event.AgentDisconnected, false);
1620+
throw new CloudRuntimeException("Unable to connect " + attache.getId(), e);
16141621
}
16151622
}
16161623
}

‎server/src/com/cloud/storage/StorageManager.java‎

Lines changed: 27 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,30 @@
1717
*/
1818
package com.cloud.storage;
1919

20-
import java.util.List;
21-
22-
import com.cloud.agent.api.Answer;
23-
import com.cloud.agent.api.Command;
24-
import com.cloud.agent.manager.Commands;
25-
import com.cloud.capacity.CapacityVO;
26-
import com.cloud.dc.DataCenterVO;
27-
import com.cloud.dc.HostPodVO;
28-
import com.cloud.deploy.DeployDestination;
29-
import com.cloud.exception.ConcurrentOperationException;
30-
import com.cloud.exception.InsufficientStorageCapacityException;
31-
import com.cloud.exception.StorageUnavailableException;
32-
import com.cloud.host.Host;
33-
import com.cloud.host.HostVO;
34-
import com.cloud.hypervisor.Hypervisor.HypervisorType;
35-
import com.cloud.service.ServiceOfferingVO;
36-
import com.cloud.storage.Volume.Type;
37-
import com.cloud.user.Account;
38-
import com.cloud.utils.Pair;
39-
import com.cloud.utils.component.Manager;
40-
import com.cloud.vm.DiskProfile;
41-
import com.cloud.vm.VMInstanceVO;
42-
import com.cloud.vm.VirtualMachine;
43-
import com.cloud.vm.VirtualMachineProfile;
20+
import java.util.List;
21+
22+
import com.cloud.agent.api.Answer;
23+
import com.cloud.agent.api.Command;
24+
import com.cloud.agent.manager.Commands;
25+
import com.cloud.capacity.CapacityVO;
26+
import com.cloud.dc.DataCenterVO;
27+
import com.cloud.dc.HostPodVO;
28+
import com.cloud.deploy.DeployDestination;
29+
import com.cloud.exception.ConcurrentOperationException;
30+
import com.cloud.exception.InsufficientStorageCapacityException;
31+
import com.cloud.exception.StorageUnavailableException;
32+
import com.cloud.host.Host;
33+
import com.cloud.host.HostVO;
34+
import com.cloud.hypervisor.Hypervisor.HypervisorType;
35+
import com.cloud.service.ServiceOfferingVO;
36+
import com.cloud.storage.Volume.Type;
37+
import com.cloud.user.Account;
38+
import com.cloud.utils.Pair;
39+
import com.cloud.utils.component.Manager;
40+
import com.cloud.vm.DiskProfile;
41+
import com.cloud.vm.VMInstanceVO;
42+
import com.cloud.vm.VirtualMachine;
43+
import com.cloud.vm.VirtualMachineProfile;
4444

4545
public interface StorageManager extends Manager {
4646
boolean canVmRestartOnAnotherServer(long vmId);
@@ -76,9 +76,10 @@ public interface StorageManager extends Manager {
7676
/**
7777
* Add a pool to a host
7878
* @param hostId
79-
* @param pool
79+
* @param pool
80+
* @throws StorageUnavailableException
8081
*/
81-
boolean addPoolToHost(long hostId, StoragePoolVO pool);
82+
void addPoolToHost(long hostId, StoragePoolVO pool) throws StorageUnavailableException;
8283

8384
/**
8485
* Moves a volume from its current storage pool to a storage pool with enough capacity in the specified zone, pod, or cluster

‎server/src/com/cloud/storage/StorageManagerImpl.java‎

Lines changed: 28 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,9 +1169,11 @@ public StoragePoolVO createPool(CreateStoragePoolCmd cmd) throws ResourceInUseEx
11691169
s_logger.debug("In createPool Adding the pool to each of the hosts");
11701170
List<HostVO> poolHosts = new ArrayList<HostVO>();
11711171
for (HostVO h : allHosts) {
1172-
success = addPoolToHost(h.getId(), pool);
1173-
if (success) {
1172+
try {
1173+
addPoolToHost(h.getId(), pool);
11741174
poolHosts.add(h);
1175+
} catch (Exception e) {
1176+
s_logger.warn("Unable to establish a connection between " + h + " and " + pool, e);
11751177
}
11761178
}
11771179

@@ -1373,45 +1375,42 @@ public boolean delPoolFromHost(long hostId) {
13731375
}
13741376

13751377
@Override
1376-
public boolean addPoolToHost(long hostId, StoragePoolVO pool) {
1378+
public void addPoolToHost(long hostId, StoragePoolVO pool) throws StorageUnavailableException {
13771379
s_logger.debug("Adding pool " + pool.getName() + " to host " + hostId);
13781380
if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem && pool.getPoolType() != StoragePoolType.IscsiLUN
13791381
&& pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && pool.getPoolType() != StoragePoolType.SharedMountPoint
13801382
&& pool.getPoolType() != StoragePoolType.PreSetup) {
1381-
s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType());
1382-
return false;
1383+
throw new CloudRuntimeException("Doesn't support storage pool type " + pool.getPoolType());
13831384
}
13841385

13851386
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool);
13861387
final Answer answer = _agentMgr.easySend(hostId, cmd);
13871388

1388-
if (answer != null) {
1389-
if (answer.getResult() == false) {
1390-
String msg = "Add host failed due to ModifyStoragePoolCommand failed" + answer.getDetails();
1391-
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg);
1392-
s_logger.warn(msg);
1393-
return false;
1394-
}
1395-
if (answer instanceof ModifyStoragePoolAnswer) {
1396-
ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer;
1397-
1398-
StoragePoolHostVO poolHost = _poolHostDao.findByPoolHost(pool.getId(), hostId);
1399-
if (poolHost == null) {
1400-
poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
1401-
_poolHostDao.persist(poolHost);
1402-
} else {
1403-
poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
1404-
}
1405-
pool.setAvailableBytes(mspAnswer.getPoolInfo().getAvailableBytes());
1406-
pool.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes());
1407-
_storagePoolDao.update(pool.getId(), pool);
1408-
return true;
1409-
}
1389+
if (answer == null) {
1390+
throw new StorageUnavailableException("Unable to get an answer to the modify storage pool command", pool.getId());
1391+
}
1392+
1393+
if (!answer.getResult()) {
1394+
String msg = "Add host failed due to ModifyStoragePoolCommand failed" + answer.getDetails();
1395+
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg);
1396+
throw new StorageUnavailableException("Unable establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails(), pool.getId());
1397+
}
1398+
1399+
assert (answer instanceof ModifyStoragePoolAnswer) : "Well, now why won't you actually return the ModifyStoragePoolAnswer when it's ModifyStoragePoolCommand? Pool=" + pool.getId() + "Host=" + hostId;
1400+
ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer;
14101401

1402+
StoragePoolHostVO poolHost = _poolHostDao.findByPoolHost(pool.getId(), hostId);
1403+
if (poolHost == null) {
1404+
poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
1405+
_poolHostDao.persist(poolHost);
14111406
} else {
1412-
return false;
1407+
poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/"));
14131408
}
1414-
return false;
1409+
pool.setAvailableBytes(mspAnswer.getPoolInfo().getAvailableBytes());
1410+
pool.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes());
1411+
_storagePoolDao.update(pool.getId(), pool);
1412+
1413+
s_logger.info("Connection established between " + pool + " host + " + hostId);
14151414
}
14161415

14171416
@Override

‎server/src/com/cloud/storage/listener/StoragePoolMonitor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.cloud.agent.api.Command;
2929
import com.cloud.agent.api.StartupCommand;
3030
import com.cloud.agent.api.StartupRoutingCommand;
31+
import com.cloud.exception.ConnectionException;
3132
import com.cloud.host.HostVO;
3233
import com.cloud.host.Status;
3334
import com.cloud.host.dao.HostDao;
@@ -66,7 +67,7 @@ public synchronized boolean processDisconnect(long agentId, Status state) {
6667
}
6768

6869
@Override
69-
public void processConnect(HostVO host, StartupCommand cmd) {
70+
public void processConnect(HostVO host, StartupCommand cmd) throws ConnectionException {
7071
if (cmd instanceof StartupRoutingCommand) {
7172
StartupRoutingCommand scCmd = (StartupRoutingCommand)cmd;
7273
if (scCmd.getHypervisorType() == HypervisorType.XenServer || scCmd.getHypervisorType() == HypervisorType.KVM ||
@@ -75,8 +76,11 @@ public void processConnect(HostVO host, StartupCommand cmd) {
7576
for (StoragePoolVO pool : pools) {
7677
Long hostId = host.getId();
7778
s_logger.debug("Host " + hostId + " connected, sending down storage pool information ...");
78-
if(_storageManager.addPoolToHost(hostId, pool)){
79+
try {
80+
_storageManager.addPoolToHost(hostId, pool);
7981
_storageManager.createCapacityEntry(pool);
82+
} catch (Exception e) {
83+
throw new ConnectionException(true, "Unable to connect to pool " + pool, e);
8084
}
8185
}
8286
}

0 commit comments

Comments
 (0)