Skip to content

Commit 9bb6cf8

Browse files
karuturikaruturi
committed
volume upload: persisting the volume metadata
 on calling GetUploadParamsForVolume, persisting the metadata to db validating the account limits and incrementing the appropriate limits encoded the metadata on management server using preshared key
1 parent 36c0c38 commit 9bb6cf8

9 files changed

Lines changed: 213 additions & 63 deletions

File tree

‎api/src/com/cloud/storage/VolumeApiService.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,17 @@
2222
import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
2323
import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd;
2424
import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd;
25+
import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd;
2526
import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
2627
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
2728
import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd;
2829

2930
import com.cloud.exception.ConcurrentOperationException;
3031
import com.cloud.exception.ResourceAllocationException;
3132
import com.cloud.user.Account;
33+
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
34+
35+
import java.net.MalformedURLException;
3236

3337
public interface VolumeApiService {
3438
/**
@@ -72,6 +76,8 @@ public interface VolumeApiService {
7276
*/
7377
Volume uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationException;
7478

79+
GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException;
80+
7581
boolean deleteVolume(long volumeId, Account caller) throws ConcurrentOperationException;
7682

7783
Volume attachVolumeToVM(AttachVolumeCmd command);

‎api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,8 @@
1919
package org.apache.cloudstack.api.command.user.volume;
2020

2121
import java.net.MalformedURLException;
22-
import java.net.URL;
23-
import java.util.UUID;
2422

23+
import com.cloud.exception.ResourceAllocationException;
2524
import org.apache.cloudstack.api.APICommand;
2625
import org.apache.cloudstack.api.AbstractGetUploadParamsCmd;
2726
import org.apache.cloudstack.api.ApiConstants;
@@ -57,18 +56,14 @@ public Long getDiskOfferingId() {
5756

5857
@Override
5958
public void execute() throws ServerApiException {
60-
// TODO Auto-generated method stub
59+
6160
try {
62-
GetUploadParamsResponse response = createGetUploadParamsResponse(
63-
UUID.fromString("C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"),
64-
new URL("https://1-2-3-4.xyz.com/upload/C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"),
65-
"TKPFeuz2nHmE/kcREEu24mnj1MrLdzOeJIHXR9HLIGgk56bkRJHaD0RRL2lds1rKKhrro4/PuleEh4YhRinhxaAmPpU4e55eprG8gTCX0ItyFAtlZViVdKXMew5Dfp4Qg8W9I1/IsDJd2Kas9/ftDQLiemAlPt0uS7Ou6asOCpifnBaKvhM4UGEjHSnni1KhBzjgEyDW3Y42HKJSSv58Sgmxl9LCewBX8vtn9tXKr+j4afj7Jlh7DFhyo9HOPC5ogR4hPBKqP7xF9tHxAyq6YqfBzsng3Xwe+Pb8TU1kFHg1l2DM4tY6ooW2h8lOhWUkrJu4hOAOeTeRtCjW3H452NKoeA1M8pKWuqMo5zRMti2u2hNZs0YY2yOy8oWMMG+lG0hvIlajqEU=",
66-
"2014-10-17T12:00:00+0530", "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9");
61+
GetUploadParamsResponse response = _volumeService.uploadVolume(this);
6762
response.setResponseName(getCommandName());
6863
setResponseObject(response);
69-
70-
} catch (MalformedURLException e) {
71-
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "malformedurl exception: " + e.getMessage());
64+
} catch (MalformedURLException | ResourceAllocationException e) {
65+
s_logger.error("exception while uploading volume", e);
66+
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "exception while uploading a volume: " + e.getMessage());
7267
}
7368
}
7469

‎core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ public TemplateOrVolumePostUploadCommand(DataObject dataObject, EndPoint endPoin
3131
this.endPoint = endPoint;
3232
}
3333

34+
public TemplateOrVolumePostUploadCommand() {
35+
}
36+
3437
public DataObject getDataObject() {
3538
return dataObject;
3639
}
@@ -56,7 +59,7 @@ public boolean equals(Object o) {
5659
return false;
5760
}
5861

59-
TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand) o;
62+
TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand)o;
6063

6164
return dataObject.equals(that.dataObject) && endPoint.equals(that.endPoint);
6265

@@ -69,10 +72,8 @@ public int hashCode() {
6972
return result;
7073
}
7174

72-
@Override public String toString() {
73-
return "TemplateOrVolumePostUploadCommand{" +
74-
"dataObject=" + dataObject +
75-
", endPoint=" + endPoint +
76-
'}';
75+
@Override
76+
public String toString() {
77+
return "TemplateOrVolumePostUploadCommand{" + "dataObject=" + dataObject + ", endPoint=" + endPoint + '}';
7778
}
7879
}

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ AsyncCallFuture<VolumeApiResult> createVolumeFromTemplateAsync(VolumeInfo volume
9494

9595
AsyncCallFuture<VolumeApiResult> registerVolume(VolumeInfo volume, DataStore store);
9696

97+
public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store);
98+
9799
AsyncCallFuture<VolumeApiResult> resize(VolumeInfo volume);
98100

99101
void resizeVolumeOnHypervisor(long volumeId, long newSize, long destHostId, String instanceName);

‎engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1223,6 +1223,20 @@ public AsyncCallFuture<VolumeApiResult> registerVolume(VolumeInfo volume, DataSt
12231223
return future;
12241224
}
12251225

1226+
@Override
1227+
public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store) {
1228+
DataObject volumeOnStore = store.create(volume);
1229+
1230+
volumeOnStore.processEvent(Event.CreateOnlyRequested);
1231+
1232+
EndPoint ep = _epSelector.select(store);
1233+
if (ep == null) {
1234+
s_logger.warn("There is no secondary storage VM for image store " + store.getName());
1235+
return null;
1236+
}
1237+
return ep;
1238+
}
1239+
12261240
protected Void registerVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeContext<VolumeApiResult> context) {
12271241
CreateCmdResult result = callback.getResult();
12281242
try {

‎server/src/com/cloud/storage/VolumeApiServiceImpl.java‎

Lines changed: 94 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
// under the License.
1717
package com.cloud.storage;
1818

19+
import java.net.MalformedURLException;
20+
import java.net.URL;
1921
import java.util.ArrayList;
22+
import java.util.Arrays;
2023
import java.util.Date;
2124
import java.util.HashMap;
2225
import java.util.List;
@@ -26,6 +29,15 @@
2629

2730
import javax.inject.Inject;
2831

32+
import com.cloud.utils.EncryptionUtil;
33+
import com.google.gson.ExclusionStrategy;
34+
import com.google.gson.FieldAttributes;
35+
import com.google.gson.Gson;
36+
import com.google.gson.GsonBuilder;
37+
import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd;
38+
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
39+
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
40+
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
2941
import org.apache.log4j.Logger;
3042

3143
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
@@ -146,6 +158,8 @@
146158
import com.cloud.vm.dao.VMInstanceDao;
147159
import com.cloud.vm.snapshot.VMSnapshotVO;
148160
import com.cloud.vm.snapshot.dao.VMSnapshotDao;
161+
import org.joda.time.DateTime;
162+
import org.joda.time.DateTimeZone;
149163

150164
public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiService, VmWorkJobHandler {
151165
private final static Logger s_logger = Logger.getLogger(VolumeApiServiceImpl.class);
@@ -261,6 +275,70 @@ public VolumeVO uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationExcep
261275
return volume;
262276
}
263277

278+
@Override
279+
@DB
280+
@ActionEvent(eventType = EventTypes.EVENT_VOLUME_UPLOAD, eventDescription = "uploading volume for post upload", async = true)
281+
public GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException {
282+
Account caller = CallContext.current().getCallingAccount();
283+
long ownerId = cmd.getEntityOwnerId();
284+
Account owner = _entityMgr.findById(Account.class, ownerId);
285+
Long zoneId = cmd.getZoneId();
286+
String volumeName = cmd.getName();
287+
String format = cmd.getFormat();
288+
Long diskOfferingId = cmd.getDiskOfferingId();
289+
String imageStoreUuid = cmd.getImageStoreUuid();
290+
DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId);
291+
292+
validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId);
293+
294+
VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, cmd.getFormat(), diskOfferingId);
295+
296+
VolumeInfo vol = volFactory.getVolume(volume.getId());
297+
298+
RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), cmd.getFormat());
299+
vol.addPayload(payload);
300+
301+
EndPoint ep = volService.registerVolumeForPostUpload(vol, store);
302+
303+
TemplateOrVolumePostUploadCommand command = new TemplateOrVolumePostUploadCommand(vol, ep);
304+
305+
GetUploadParamsResponse response = new GetUploadParamsResponse();
306+
String url = "https://" + command.getEndPoint().getPublicAddr() + "/upload/" + command.getDataObject().getUuid();
307+
response.setPostURL(new URL(url));
308+
309+
response.setId(UUID.fromString(command.getDataObject().getUuid()));
310+
311+
/*
312+
* TODO: hardcoding the timeout to current + 60 min for now. This needs to goto the database
313+
*/
314+
DateTime currentDateTime = new DateTime(DateTimeZone.UTC);
315+
currentDateTime.plusHours(1);
316+
String expires = currentDateTime.toString();
317+
response.setTimeout(expires);
318+
319+
String key = _configDao.getValue(Config.SSVMPSK.key());
320+
/*
321+
* encoded metadata using the post upload config ssh key
322+
*/
323+
final List<String> fieldExclusions = Arrays.asList("s_logger");
324+
Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() {
325+
@Override public boolean shouldSkipField(FieldAttributes f) {
326+
return f.getDeclaringClass() == Logger.class;
327+
}
328+
@Override public boolean shouldSkipClass(Class<?> clazz) {
329+
return false;
330+
}
331+
}).create();
332+
String jsonPayload = gson.toJson(command);
333+
response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key));
334+
335+
/*
336+
* signature calculated on the url, expiry, metadata.
337+
*/
338+
response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key));
339+
return response;
340+
}
341+
264342
private boolean validateVolume(Account caller, long ownerId, Long zoneId, String volumeName, String url,
265343
String format, Long diskOfferingId) throws ResourceAllocationException {
266344

@@ -282,21 +360,23 @@ private boolean validateVolume(Account caller, long ownerId, Long zoneId, String
282360
throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + zoneId);
283361
}
284362

285-
if (url.toLowerCase().contains("file://")) {
286-
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
363+
//validating the url only when url is not null. url can be null incase of form based post upload
364+
if (url != null ) {
365+
if( url.toLowerCase().contains("file://")) {
366+
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
367+
}
368+
UriUtils.validateUrl(format, url);
369+
// Check that the resource limit for secondary storage won't be exceeded
370+
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
371+
} else {
372+
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
287373
}
288374

289375
ImageFormat imgfmt = ImageFormat.valueOf(format.toUpperCase());
290376
if (imgfmt == null) {
291377
throw new IllegalArgumentException("Image format is incorrect " + format + ". Supported formats are " + EnumUtils.listValues(ImageFormat.values()));
292378
}
293379

294-
UriUtils.validateUrl(format, url);
295-
296-
297-
// Check that the resource limit for secondary storage won't be exceeded
298-
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
299-
300380
// Check that the the disk offering specified is valid
301381
if (diskOfferingId != null) {
302382
DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId);
@@ -357,7 +437,12 @@ public VolumeVO doInTransaction(TransactionStatus status) {
357437
// Increment resource count during allocation; if actual creation fails,
358438
// decrement it
359439
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
360-
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
440+
//url can be null incase of postupload
441+
if(url!=null) {
442+
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
443+
} else {
444+
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage);
445+
}
361446

362447
return volume;
363448
}

‎server/src/com/cloud/template/TemplateManagerImpl.java‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
import java.net.MalformedURLException;
2020
import java.net.URISyntaxException;
2121
import java.net.URL;
22-
import java.security.InvalidKeyException;
23-
import java.security.NoSuchAlgorithmException;
2422
import java.util.ArrayList;
2523
import java.util.Date;
2624
import java.util.HashMap;
@@ -31,17 +29,18 @@
3129
import java.util.concurrent.ExecutorService;
3230
import java.util.concurrent.Executors;
3331

34-
import javax.crypto.Mac;
35-
import javax.crypto.spec.SecretKeySpec;
3632
import javax.ejb.Local;
3733
import javax.inject.Inject;
3834
import javax.naming.ConfigurationException;
3935

36+
import com.cloud.utils.EncryptionUtil;
37+
import com.google.gson.ExclusionStrategy;
38+
import com.google.gson.FieldAttributes;
4039
import com.google.gson.Gson;
40+
import com.google.gson.GsonBuilder;
4141
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
4242
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
4343
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
44-
import org.apache.commons.codec.binary.Base64;
4544
import org.apache.commons.collections.CollectionUtils;
4645
import org.apache.log4j.Logger;
4746

@@ -363,40 +362,32 @@ public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForT
363362
String expires = currentDateTime.toString();
364363
response.setTimeout(expires);
365364

365+
String key = _configDao.getValue(Config.SSVMPSK.key());
366366
/*
367367
* encoded metadata using the post upload config ssh key
368368
*/
369-
Gson gson = new Gson();
369+
Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() {
370+
@Override public boolean shouldSkipField(FieldAttributes f) {
371+
return f.getDeclaredType().getClass().isInstance(Logger.class);
372+
}
373+
@Override public boolean shouldSkipClass(Class<?> clazz) {
374+
return false;
375+
}
376+
}).create();
370377
String jsonPayload = gson.toJson(payload);
371-
response.setMetadata(encodeData(jsonPayload));
378+
response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key));
372379

373380
/*
374381
* signature calculated on the url, expiry, metadata.
375382
*/
376-
response.setSignature(encodeData(jsonPayload+url+expires));
383+
response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key));
377384

378385
return response;
379386
} else {
380387
return null;
381388
}
382389
}
383390

384-
private String encodeData(String data) {
385-
String key = _configDao.getValue(Config.SSVMPSK.key());
386-
387-
try {
388-
final Mac mac = Mac.getInstance("HmacSHA1");
389-
final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1");
390-
mac.init(keySpec);
391-
mac.update(data.getBytes());
392-
final byte[] encryptedBytes = mac.doFinal();
393-
final String computedSignature = Base64.encodeBase64String(encryptedBytes);
394-
return computedSignature;
395-
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
396-
s_logger.error("exception occured which encoding the data.", e);
397-
return null;
398-
}
399-
}
400391

401392
@Override
402393
public DataStore getImageStore(String storeUuid, Long zoneId) {

‎utils/pom.xml‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -183,19 +183,6 @@
183183
</excludes>
184184
</configuration>
185185
</plugin>
186-
<plugin>
187-
<groupId>com.mycila</groupId>
188-
<artifactId>license-maven-plugin</artifactId>
189-
<executions>
190-
<execution>
191-
<id>cloudstack-checklicence</id>
192-
<phase>process-classes</phase>
193-
<goals>
194-
<goal>check</goal>
195-
</goals>
196-
</execution>
197-
</executions>
198-
</plugin>
199186
</plugins>
200187
<resources>
201188
<resource>

0 commit comments

Comments
 (0)