Skip to content

Improve logs on KVMHostActivityChecker #4706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DaanHoogland merged 2 commits into from
Apr 13, 2021
Merged

Improve logs on KVMHostActivityChecker #4706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4cfa7a4
Improve logs on KVMHostActivityChecker
Feb 18, 2021
971b047
Add more host information in log
Feb 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/schema/src/main/java/com/cloud/host/HostVO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ public boolean equals(Object obj) {

@Override
public String toString() {
return new StringBuilder("Host[").append("-").append(id).append("-").append(type).append("]").toString();
return String.format("Host [{id: \"%s\", name: \"%s\", uuid: \"%s\", type=\"%s\"}]", id, name, uuid, type);
}

public void setHypervisorType(HypervisorType hypervisorType) {
Expand Down
86 changes: 55 additions & 31 deletions ...ns/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,13 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck
public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException {
try {
return isVMActivtyOnHost(r, suspectTime);
}
catch (StorageUnavailableException e){
throw new HACheckerException("Storage is unavailable to do the check, mostly host is not reachable ", e);
}
catch (Exception e){
throw new HACheckerException("Operation timed out, mostly host is not reachable ", e);
} catch (HACheckerException e) {
//Re-throwing the exception to avoid poluting the 'HACheckerException' already thrown
throw e;
} catch (Exception e){
String message = String.format("Operation timed out, probably the %s is not reachable.", r.toString());
LOG.warn(message, e);
throw new HACheckerException(message, e);
}
}

Expand All @@ -85,49 +86,53 @@ public boolean isHealthy(Host r) {

private boolean isAgentActive(Host agent) {
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
throw new IllegalStateException("Calling KVM investigator for non KVM Host of type " + agent.getHypervisorType());
throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType()));
}
Status hostStatus = Status.Unknown;
Status neighbourStatus = Status.Unknown;
final CheckOnHostCommand cmd = new CheckOnHostCommand(agent);
try {
LOG.debug(String.format("Checking %s status...", agent.toString()));
Answer answer = agentMgr.easySend(agent.getId(), cmd);
if (answer != null) {
hostStatus = answer.getResult() ? Status.Down : Status.Up;
LOG.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus));

if ( hostStatus == Status.Up ){
return true;
}
}
else {
LOG.debug(String.format("Setting %s to \"Disconnected\" status.", agent.toString()));
hostStatus = Status.Disconnected;
}
} catch (Exception e) {
LOG.warn("Failed to send command to host: " + agent.getId());
LOG.warn(String.format("Failed to send command CheckOnHostCommand to %s.", agent.toString()), e);
}

List<HostVO> neighbors = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
for (HostVO neighbor : neighbors) {
if (neighbor.getId() == agent.getId() || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
continue;
}
if (LOG.isTraceEnabled()){
LOG.trace("Investigating host:" + agent.getId() + " via neighbouring host:" + neighbor.getId());
}

try {
LOG.debug(String.format("Investigating %s via neighbouring %s.", agent.toString(), neighbor.toString()));

Answer answer = agentMgr.easySend(neighbor.getId(), cmd);
if (answer != null) {
neighbourStatus = answer.getResult() ? Status.Down : Status.Up;
if (LOG.isTraceEnabled()){
LOG.trace("Neighbouring host:" + neighbor.getId() + " returned status:" + neighbourStatus + " for the investigated host:" + agent.getId());
}

LOG.debug(String.format("Neighbouring %s returned status [%s] for the investigated %s.", neighbor.toString(), neighbourStatus, agent.toString()));

if (neighbourStatus == Status.Up) {
break;
}
} else {
LOG.debug(String.format("Neighbouring %s is Disconnected.", neighbor.toString()));
}
} catch (Exception e) {
if (LOG.isTraceEnabled()) {
LOG.trace("Failed to send command to host: " + neighbor.getId());
}
LOG.warn(String.format("Failed to send command CheckOnHostCommand to %s.", neighbor.toString()), e);
}
}
if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
Expand All @@ -137,32 +142,48 @@ private boolean isAgentActive(Host agent) {
hostStatus = Status.Down;
}

if (LOG.isTraceEnabled()){
LOG.trace("Resource state = " + hostStatus.name());
}
LOG.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus));

return hostStatus == Status.Up;
}

private boolean isVMActivtyOnHost(Host agent, DateTime suspectTime) throws StorageUnavailableException {
private boolean isVMActivtyOnHost(Host agent, DateTime suspectTime) throws HACheckerException {
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
throw new IllegalStateException("Calling KVM investigator for non KVM Host of type " + agent.getHypervisorType());
throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType()));
}
boolean activityStatus = true;
HashMap<StoragePool, List<Volume>> poolVolMap = getVolumeUuidOnHost(agent);
for (StoragePool pool : poolVolMap.keySet()) {
//for each storage pool find activity
List<Volume> volume_list = poolVolMap.get(pool);
final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime);
//send the command to appropriate storage pool
activityStatus = verifyActivityOfStorageOnHost(poolVolMap, pool, agent, suspectTime, activityStatus);
if (!activityStatus) {
LOG.warn(String.format("It seems that the storage pool [%s] does not have activity on %s.", pool.getId(), agent.toString()));
break;
}
}

return activityStatus;
}

protected boolean verifyActivityOfStorageOnHost(HashMap<StoragePool, List<Volume>> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException {
List<Volume> volume_list = poolVolMap.get(pool);
final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime);

LOG.debug(String.format("Checking VM activity for %s on storage pool [%s].", agent.toString(), pool.getId()));
try {
Answer answer = storageManager.sendToPool(pool, getNeighbors(agent), cmd);

if (answer != null) {
activityStatus = ! answer.getResult();
activityStatus = !answer.getResult();
LOG.debug(String.format("%s %s activity on storage pool [%s]", agent.toString(), activityStatus ? "has" : "does not have", pool.getId()));
} else {
throw new IllegalStateException("Did not get a valid response for VM activity check for host " + agent.getId());
String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", agent.toString(), pool.getId());
LOG.debug(message);
throw new IllegalStateException(message);
}
}
if (LOG.isDebugEnabled()){
LOG.debug("Resource active = " + activityStatus);
} catch (StorageUnavailableException e){
String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool.getId(), agent.toString());
LOG.warn(message, e);
throw new HACheckerException(message, e);
}
return activityStatus;
}
Expand All @@ -171,12 +192,14 @@ private HashMap<StoragePool, List<Volume>> getVolumeUuidOnHost(Host agent) {
List<VMInstanceVO> vm_list = vmInstanceDao.listByHostId(agent.getId());
List<VolumeVO> volume_list = new ArrayList<VolumeVO>();
for (VirtualMachine vm : vm_list) {
LOG.debug(String.format("Retrieving volumes of VM [%s]...", vm.getId()));
List<VolumeVO> vm_volume_list = volumeDao.findByInstance(vm.getId());
volume_list.addAll(vm_volume_list);
}

HashMap<StoragePool, List<Volume>> poolVolMap = new HashMap<StoragePool, List<Volume>>();
for (Volume vol : volume_list) {
LOG.debug(String.format("Retrieving storage pool [%s] of volume [%s]...", vol.getPoolId(), vol.getId()));
StoragePool sp = storagePool.findById(vol.getPoolId());
if (!poolVolMap.containsKey(sp)) {
List<Volume> list = new ArrayList<Volume>();
Expand All @@ -193,6 +216,7 @@ private HashMap<StoragePool, List<Volume>> getVolumeUuidOnHost(Host agent) {
public long[] getNeighbors(Host agent) {
List<Long> neighbors = new ArrayList<Long>();
List<HostVO> cluster_hosts = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
LOG.debug(String.format("Retrieving all \"Up\" hosts from cluster [%s]...", agent.getClusterId()));
for (HostVO host : cluster_hosts) {
if (host.getId() == agent.getId() || (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
continue;
Expand Down