Skip to content

Commit 627f5a6

Browse files
koushik-daskoushik-das
committed
volume upload: management server polling and upload status from agent
MS polling logic to query status for volumes that are uploaded
1 parent c2cf250 commit 627f5a6

12 files changed

Lines changed: 442 additions & 7 deletions

File tree

‎api/src/com/cloud/storage/Volume.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,12 @@ enum State {
4545
Destroy("The volume is destroyed, and can't be recovered."),
4646
Destroying("The volume is destroying, and can't be recovered."),
4747
UploadOp("The volume upload operation is in progress or in short the volume is on secondary storage"),
48-
Uploading("volume is uploading"),
49-
Copying("volume is copying from image store to primary, in case it's an uploaded volume"),
50-
Uploaded("volume is uploaded");
48+
Copying("Volume is copying from image store to primary, in case it's an uploaded volume"),
49+
Uploaded("Volume is uploaded"),
50+
NotUploaded("The volume entry is just created in DB, not yet uploaded"),
51+
UploadInProgress("Volume upload is in progress"),
52+
UploadError("Volume upload encountered some error"),
53+
UploadAbandoned("Volume upload is abandoned since the upload was never initiated within a specificed time");
5154

5255
String _description;
5356

@@ -100,7 +103,12 @@ public String getDescription() {
100103
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Uploaded, Event.DestroyRequested, Destroy, null));
101104
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Expunged, Event.ExpungingRequested, Expunged, null));
102105
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Expunged, Event.OperationSucceeded, Expunged, null));
103-
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Expunged, Event.OperationFailed, Expunged,null));
106+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Expunged, Event.OperationFailed, Expunged, null));
107+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(NotUploaded, Event.OperationTimeout, UploadAbandoned, null));
108+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(NotUploaded, Event.UploadRequested, UploadInProgress, null));
109+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(UploadInProgress, Event.OperationSucceeded, Uploaded, null));
110+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(UploadInProgress, Event.OperationFailed, UploadError, null));
111+
s_fsm.addTransition(new StateMachine2.Transition<State, Event>(UploadInProgress, Event.OperationTimeout, UploadError, null));
104112
}
105113
}
106114

@@ -120,7 +128,8 @@ enum Event {
120128
SnapshotRequested,
121129
DestroyRequested,
122130
ExpungingRequested,
123-
ResizeRequested;
131+
ResizeRequested,
132+
OperationTimeout;
124133
}
125134

126135
/**

‎core/src/org/apache/cloudstack/storage/command/UploadStatusAnswer.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package org.apache.cloudstack.storage.command;
21+
22+
import com.cloud.agent.api.Answer;
23+
24+
public class UploadStatusAnswer extends Answer {
25+
public static enum UploadStatus {
26+
UNKNOWN, IN_PROGRESS, COMPLETED, ERROR
27+
}
28+
29+
private UploadStatus status;
30+
31+
protected UploadStatusAnswer() {
32+
}
33+
34+
public UploadStatusAnswer(UploadStatusCommand cmd, UploadStatus status, String msg) {
35+
super(cmd, false, msg);
36+
this.status = UploadStatus.ERROR;
37+
}
38+
39+
public UploadStatusAnswer(UploadStatusCommand cmd, Exception e) {
40+
super(cmd, false, e.getMessage());
41+
this.status = UploadStatus.ERROR;
42+
}
43+
44+
public UploadStatusAnswer(UploadStatusCommand cmd, UploadStatus status) {
45+
super(cmd, true, null);
46+
this.status = status;
47+
}
48+
49+
public UploadStatus getStatus() {
50+
return status;
51+
}
52+
53+
}

‎core/src/org/apache/cloudstack/storage/command/UploadStatusCommand.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package org.apache.cloudstack.storage.command;
21+
22+
import com.cloud.agent.api.Command;
23+
24+
public class UploadStatusCommand extends Command {
25+
26+
private long entityId;
27+
28+
protected UploadStatusCommand() {
29+
}
30+
31+
public UploadStatusCommand(long entityId) {
32+
this.entityId = entityId;
33+
}
34+
35+
public long getEntityId() {
36+
return entityId;
37+
}
38+
39+
@Override
40+
public boolean executeInSequence() {
41+
return false;
42+
}
43+
44+
}

‎engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
2222
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
2323

24+
import com.cloud.storage.Volume;
2425
import com.cloud.utils.db.GenericDao;
2526
import com.cloud.utils.fsm.StateDao;
2627

@@ -48,4 +49,6 @@ public interface VolumeDataStoreDao extends GenericDao<VolumeDataStoreVO, Long>,
4849
void expireDnldUrlsForZone(Long dcId);
4950

5051
List<VolumeDataStoreVO> listUploadedVolumesByStoreId(long id);
52+
53+
List<VolumeDataStoreVO> listByVolumeState(Volume.State... states);
5154
}

‎engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727

2828
import org.apache.log4j.Logger;
2929
import org.springframework.stereotype.Component;
30-
3130
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
3231
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
3332
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -36,7 +35,11 @@
3635
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
3736
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
3837

38+
import com.cloud.storage.Volume;
39+
import com.cloud.storage.VolumeVO;
40+
import com.cloud.storage.dao.VolumeDao;
3941
import com.cloud.utils.db.GenericDaoBase;
42+
import com.cloud.utils.db.JoinBuilder.JoinType;
4043
import com.cloud.utils.db.SearchBuilder;
4144
import com.cloud.utils.db.SearchCriteria;
4245
import com.cloud.utils.db.SearchCriteria.Op;
@@ -53,11 +56,15 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
5356
private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
5457
private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
5558
private SearchBuilder<VolumeDataStoreVO> uploadVolumeSearch;
59+
private SearchBuilder<VolumeVO> volumeOnlySearch;
60+
private SearchBuilder<VolumeDataStoreVO> uploadVolumeStateSearch;
5661
private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update volume_store_ref set download_url_created=? where store_id in (select id from image_store where data_center_id=?)";
5762

5863

5964
@Inject
6065
DataStoreManager storeMgr;
66+
@Inject
67+
VolumeDao volumeDao;
6168

6269
@Override
6370
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@@ -102,6 +109,13 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
102109
uploadVolumeSearch.and("destroyed", uploadVolumeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
103110
uploadVolumeSearch.done();
104111

112+
volumeOnlySearch = volumeDao.createSearchBuilder();
113+
volumeOnlySearch.and("state", volumeOnlySearch.entity().getState(), Op.IN);
114+
uploadVolumeStateSearch = createSearchBuilder();
115+
uploadVolumeStateSearch.join("volumeOnlySearch", volumeOnlySearch, volumeOnlySearch.entity().getId(), uploadVolumeStateSearch.entity().getVolumeId(), JoinType.LEFT);
116+
uploadVolumeStateSearch.and("destroyed", uploadVolumeStateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
117+
uploadVolumeStateSearch.done();
118+
105119
return true;
106120
}
107121

@@ -304,4 +318,13 @@ public void expireDnldUrlsForZone(Long dcId){
304318
}
305319

306320
}
321+
322+
@Override
323+
public List<VolumeDataStoreVO> listByVolumeState(Volume.State... states) {
324+
SearchCriteria<VolumeDataStoreVO> sc = uploadVolumeStateSearch.create();
325+
sc.setJoinParameters("volumeOnlySearch", "state", (Object[])states);
326+
sc.setParameters("destroyed", false);
327+
return listIncludingRemovedBy(sc);
328+
}
329+
307330
}

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManager.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.apache.cloudstack.storage.command.DeleteCommand;
2020
import org.apache.cloudstack.storage.command.DownloadCommand;
2121
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
22+
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
23+
import org.apache.cloudstack.storage.command.UploadStatusCommand;
2224

2325
import com.cloud.agent.api.Answer;
2426
import com.cloud.agent.api.AttachIsoCommand;
@@ -106,4 +108,7 @@ public interface MockStorageManager extends Manager {
106108
StoragePoolInfo getLocalStorage(String hostGuid, Long storageSize);
107109

108110
CopyVolumeAnswer CopyVolume(CopyVolumeCommand cmd);
111+
112+
public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd);
113+
109114
}

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockStorageManagerImpl.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@
3333

3434
import org.apache.log4j.Logger;
3535
import org.springframework.stereotype.Component;
36-
3736
import org.apache.cloudstack.storage.command.DeleteCommand;
3837
import org.apache.cloudstack.storage.command.DownloadCommand;
3938
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
39+
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
40+
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
41+
import org.apache.cloudstack.storage.command.UploadStatusCommand;
4042

4143
import com.cloud.agent.api.Answer;
4244
import com.cloud.agent.api.AttachIsoCommand;
@@ -1255,4 +1257,9 @@ public CopyVolumeAnswer CopyVolume(CopyVolumeCommand cmd) {
12551257
return new CopyVolumeAnswer(cmd, true, null, primaryStorage.getMountPoint(), vol.getPath());
12561258
}
12571259
}
1260+
1261+
@Override
1262+
public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd) {
1263+
return new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN);
1264+
}
12581265
}

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.cloudstack.storage.command.DownloadCommand;
3636
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
3737
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
38+
import org.apache.cloudstack.storage.command.UploadStatusCommand;
3839

3940
import com.cloud.agent.api.Answer;
4041
import com.cloud.agent.api.AttachIsoCommand;
@@ -371,6 +372,8 @@ public Answer simulate(Command cmd, String hostGuid) {
371372
answer = _mockStorageMgr.ComputeChecksum((ComputeChecksumCommand)cmd);
372373
} else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) {
373374
answer = _mockStorageMgr.CreatePrivateTemplateFromVolume((CreatePrivateTemplateFromVolumeCommand)cmd);
375+
} else if (cmd instanceof UploadStatusCommand) {
376+
answer = _mockStorageMgr.getUploadStatus((UploadStatusCommand)cmd);
374377
} else if (cmd instanceof MaintainCommand) {
375378
answer = _mockAgentMgr.maintain((MaintainCommand)cmd);
376379
} else if (cmd instanceof GetVmStatsCommand) {

‎server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,4 +264,5 @@
264264
<property name="gslbServiceProviders" value="#{gslbServiceProvidersRegistry.registered}" />
265265
</bean>
266266
<bean id="certServiceImpl" class="org.apache.cloudstack.network.lb.CertServiceImpl" />
267+
<bean id="imageStoreUploadMonitorImpl" class="com.cloud.storage.ImageStoreUploadMonitorImpl" />
267268
</beans>

‎server/src/com/cloud/storage/ImageStoreUploadMonitor.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.storage;
18+
19+
import com.cloud.utils.component.Manager;
20+
21+
/**
22+
* Monitor upload progress of all entities.
23+
*
24+
*/
25+
public interface ImageStoreUploadMonitor extends Manager {
26+
27+
}

0 commit comments

Comments
 (0)