Skip to content

Add ability to download templates in Swift #1332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
asfgit merged 1 commit into from
Apr 6, 2016
Merged

Add ability to download templates in Swift #1332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions ...e/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,15 @@
*/
package org.apache.cloudstack.storage.datastore.driver;

import java.net.URL;
import java.util.Map;
import java.util.UUID;

import javax.inject.Inject;

import com.cloud.configuration.Config;
import com.cloud.utils.SwiftUtil;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.log4j.Logger;

import org.apache.cloudstack.api.ApiConstants;
Expand All @@ -43,7 +48,6 @@
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.UnsupportedServiceException;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.utils.exception.CloudRuntimeException;
Expand All @@ -57,6 +61,8 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
EndPointSelector _epSelector;
@Inject
StorageCacheManager cacheManager;
@Inject
ConfigurationDao _configDao;

@Override
public DataStoreTO getStoreTO(DataStore store) {
Expand All @@ -67,7 +73,29 @@ public DataStoreTO getStoreTO(DataStore store) {

@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
throw new UnsupportedServiceException("Extract entity url is not yet supported for Swift image store provider");

SwiftTO swiftTO = (SwiftTO)store.getTO();
String tempKey = UUID.randomUUID().toString();
boolean result = SwiftUtil.setTempKey(swiftTO, tempKey);

if (!result) {
String errMsg = "Unable to set Temp-Key: " + tempKey;
s_logger.error(errMsg);
throw new CloudRuntimeException(errMsg);
}

String containerName = SwiftUtil.getContainerName(dataObject.getType().toString(), dataObject.getId());
String objectName = installPath.split("\\/")[1];
// Get extract url expiration interval set in global configuration (in seconds)
int urlExpirationInterval = Integer.parseInt(_configDao.getValue(Config.ExtractURLExpirationInterval.toString()));

URL swiftUrl = SwiftUtil.generateTempUrl(swiftTO, containerName, objectName, tempKey, urlExpirationInterval);
if (swiftUrl != null) {
s_logger.debug("Swift temp-url: " + swiftUrl.toString());
return swiftUrl.toString();
}

Copy link
Contributor

@pdube pdube Jan 13, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could instead add a log statement here that indicates that the extraction url could not be created for the container and object

throw new CloudRuntimeException("Unable to create extraction URL");
}

@Override
Expand Down
6 changes: 0 additions & 6 deletions server/src/com/cloud/template/TemplateManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1520,12 +1520,6 @@ public VirtualMachineTemplate createPrivateTemplate(CreateTemplateCmd command) t
}

} finally {
/*if (snapshot != null && snapshot.getSwiftId() != null
&& secondaryStorageURL != null && zoneId != null
&& accountId != null && volumeId != null) {
_snapshotMgr.deleteSnapshotsForVolume(secondaryStorageURL,
zoneId, accountId, volumeId);
}*/
if (privateTemplate == null) {
final VolumeVO volumeFinal = volume;
final SnapshotVO snapshotFinal = snapshot;
Expand Down
76 changes: 71 additions & 5 deletions utils/src/main/java/com/cloud/utils/SwiftUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,30 @@
package com.cloud.utils;

import java.io.File;
import java.util.Arrays;
import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.util.Map;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Formatter;

import org.apache.log4j.Logger;

import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;

public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";



public interface SwiftClientCfg {
String getAccount();
Expand Down Expand Up @@ -143,8 +155,7 @@ public static String[] list(SwiftClientCfg swift, String container, String rFile
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
String[] lines = parser.getLines().split("\\n");
return lines;
return parser.getLines().split("\\n");
} else {
if (result != null) {
String errMsg = "swiftList failed , err=" + result;
Expand All @@ -161,7 +172,7 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
String container = swiftPath.substring(0, firstIndexOfSeparator);
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
String destFilePath = null;
String destFilePath;
if (destDirectory.isDirectory()) {
destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
} else {
Expand All @@ -171,7 +182,7 @@ public static File getObject(SwiftClientCfg cfg, File destDirectory, String swif
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
" download " + container + " " + srcPath + " -o " + destFilePath);
" download " + container + " " + srcPath + " -o " + destFilePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
Expand Down Expand Up @@ -236,4 +247,59 @@ public static boolean deleteObject(SwiftClientCfg cfg, String path) {
command.execute(parser);
return true;
}

public static boolean setTempKey(SwiftClientCfg cfg, String tempKey){
Copy link
Contributor

@wido wido Jan 27, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you write a UnitTest for this?

Copy link
Contributor Author

@syed syed Jan 28, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think I can write a unit test for this as it relies on an external script to work to give the correct output. I could mock the output but that would defeat the purpose of the unit test.


Map<String, String> tempKeyMap = new HashMap<>();
tempKeyMap.put("Temp-URL-Key", tempKey);
return postMeta(cfg, "", "", tempKeyMap);

}

public static URL generateTempUrl(SwiftClientCfg cfg, String container, String object, String tempKey, int urlExpirationInterval) {
Copy link
Contributor

@wido wido Jan 27, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here. This method is perfectly suited for a Unit Test

Copy link
Contributor Author

@syed syed Jan 28, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done


int currentTime = (int) (System.currentTimeMillis() / 1000L);
int expirationSeconds = currentTime + urlExpirationInterval;

try {

URL endpoint = new URL(cfg.getEndPoint());
String method = "GET";
String path = String.format("/v1/AUTH_%s/%s/%s", cfg.getAccount(), container, object);

//sign the request
String hmacBody = String.format("%s\n%d\n%s", method, expirationSeconds, path);
String signature = calculateRFC2104HMAC(hmacBody, tempKey);
path += String.format("?temp_url_sig=%s&temp_url_expires=%d", signature, expirationSeconds);

//generate the temp url
URL tempUrl = new URL(endpoint.getProtocol(), endpoint.getHost(), endpoint.getPort(), path);

return tempUrl;

} catch (Exception e) {
logger.error(e.getMessage());
throw new CloudRuntimeException(e.getMessage());
}

}

public static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {

SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
mac.init(signingKey);
return toHexString(mac.doFinal(data.getBytes()));

}

public static String toHexString(byte[] bytes) {

Formatter formatter = new Formatter();
for (byte b : bytes) {
formatter.format("%02x", b);
}
return formatter.toString();
}
}
92 changes: 92 additions & 0 deletions utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.utils;

import org.junit.Test;
import org.mockito.Mockito;

import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;


public class SwiftUtilTest {

@Test
public void testCalculateRFC2104HMAC() throws NoSuchAlgorithmException, SignatureException, InvalidKeyException {
String inputData = "testData";
String inputKey = "testKey";
String expected = "1d541ecb5cdb2d850716bfd55585e20a1cd8984b";
String output = SwiftUtil.calculateRFC2104HMAC(inputData, inputKey);

assertEquals(expected, output);
}

@Test
public void testToHexString(){
final byte[] input = "testing".getBytes();
final String expected = "74657374696e67";
final String result = SwiftUtil.toHexString(input);
assertEquals(expected, result);
}

@Test
public void testGenerateTempUrl() {

SwiftUtil.SwiftClientCfg cfg = Mockito.mock(SwiftUtil.SwiftClientCfg.class);
when(cfg.getEndPoint()).thenReturn("http://localhost:8080/v1/");
when(cfg.getAccount()).thenReturn("test");

String container = "testContainer";
String object = "testObject";
String tempKey = "testKey";
int urlExpirationInterval = 3600;
String expected = "http://localhost:8080/v1/AUTH_test/testContainer/testObject";
URL output = SwiftUtil.generateTempUrl(cfg, container, object, tempKey, urlExpirationInterval);

assertTrue(output.toString().contains(expected));
}

@Test
public void testSplitSwiftPath(){
String input = "container/object";
String[] output = SwiftUtil.splitSwiftPath(input);
String[] expected = {"container", "object"};

assertArrayEquals(expected, output);
}

@Test
public void testGetContainerName(){

String inputType = "Template";
long inputId = 45;
String output = SwiftUtil.getContainerName(inputType, inputId);
String expected = "T-45";

assertEquals(expected, output);
}
}