diff --git a/api/src/com/cloud/template/TemplateApiService.java b/api/src/com/cloud/template/TemplateApiService.java index 9a2e2159dc80..ea8eaf863bff 100755 --- a/api/src/com/cloud/template/TemplateApiService.java +++ b/api/src/com/cloud/template/TemplateApiService.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.template; +import java.net.MalformedURLException; import java.net.URISyntaxException; import java.util.List; @@ -29,6 +30,7 @@ import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplate; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; @@ -37,11 +39,14 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; public interface TemplateApiService { VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws URISyntaxException, ResourceAllocationException; + public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplate cmd) throws ResourceAllocationException, MalformedURLException; + VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; diff --git a/api/src/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplate.java b/api/src/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplate.java index 2b7ee32c2b0d..bb3b95898d97 100644 --- a/api/src/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplate.java +++ b/api/src/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplate.java @@ -19,11 +19,10 @@ package org.apache.cloudstack.api.command.user.template; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Map; -import java.util.UUID; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.AbstractGetUploadParamsCommand; import org.apache.cloudstack.api.ApiConstants; @@ -109,27 +108,27 @@ public Map getDetails() { return params; } - public Boolean getIsDynamicallyScalable() { + public Boolean isDynamicallyScalable() { return isDynamicallyScalable; } - public Boolean getExtractable() { + public Boolean isExtractable() { return extractable; } - public Boolean getFeatured() { + public Boolean isFeatured() { return featured; } - public Boolean getPublicTemplate() { + public Boolean isPublic() { return publicTemplate; } - public Boolean getIsRoutingType() { + public Boolean isRoutingType() { return isRoutingType; } - public Boolean getPasswordEnabled() { + public Boolean isPasswordEnabled() { return passwordEnabled; } @@ -137,7 +136,7 @@ public Boolean getRequiresHvm() { return requiresHvm; } - public Boolean getSshKeyEnabled() { + public Boolean isSshKeyEnabled() { return sshKeyEnabled; } @@ -147,17 +146,13 @@ public String getTemplateTag() { @Override public void execute() throws ServerApiException { - // TODO Auto-generated method stub try { - GetUploadParamsResponse response = createGetUploadParamsResponse( - UUID.fromString("C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"), - new URL("https://1-2-3-4.xyz.com/upload/C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"), - "TKPFeuz2nHmE/kcREEu24mnj1MrLdzOeJIHXR9HLIGgk56bkRJHaD0RRL2lds1rKKhrro4/PuleEh4YhRinhxaAmPpU4e55eprG8gTCX0ItyFAtlZViVdKXMew5Dfp4Qg8W9I1/IsDJd2Kas9/ftDQLiemAlPt0uS7Ou6asOCpifnBaKvhM4UGEjHSnni1KhBzjgEyDW3Y42HKJSSv58Sgmxl9LCewBX8vtn9tXKr+j4afj7Jlh7DFhyo9HOPC5ogR4hPBKqP7xF9tHxAyq6YqfBzsng3Xwe+Pb8TU1kFHg1l2DM4tY6ooW2h8lOhWUkrJu4hOAOeTeRtCjW3H452NKoeA1M8pKWuqMo5zRMti2u2hNZs0YY2yOy8oWMMG+lG0hvIlajqEU=", - "2014-10-17T12:00:00+0530", "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"); + GetUploadParamsResponse response = _templateService.registerTemplateForPostUpload(this); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (MalformedURLException e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "malformedurl exception: " + e.getMessage()); + } catch (ResourceAllocationException | MalformedURLException e) { + s_logger.error("exception while registering template", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "exception while registering template: " + e.getMessage()); } } diff --git a/api/src/org/apache/cloudstack/api/response/GetUploadParamsResponse.java b/api/src/org/apache/cloudstack/api/response/GetUploadParamsResponse.java index e379643e5961..9c81c08cca10 100644 --- a/api/src/org/apache/cloudstack/api/response/GetUploadParamsResponse.java +++ b/api/src/org/apache/cloudstack/api/response/GetUploadParamsResponse.java @@ -57,6 +57,9 @@ public GetUploadParamsResponse(UUID id, URL postURL, String metadata, String tim setObjectName("getuploadparams"); } + public GetUploadParamsResponse() { + } + public void setId(UUID id) { this.id = id; } diff --git a/core/pom.xml b/core/pom.xml index 0f08e0874614..cce3dc07615c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -55,21 +55,7 @@ org.apache.maven.plugins maven-pmd-plugin - - com.mycila - license-maven-plugin - - - cloudstack-checklicence - process-classes - - check - - - - - diff --git a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java new file mode 100644 index 000000000000..de943c4680c0 --- /dev/null +++ b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.command; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; + +public class TemplateOrVolumePostUploadCommand { + DataObject dataObject; + EndPoint endPoint; + + public TemplateOrVolumePostUploadCommand(DataObject dataObject, EndPoint endPoint) { + this.dataObject = dataObject; + this.endPoint = endPoint; + } + + public DataObject getDataObject() { + return dataObject; + } + + public void setDataObject(DataObject dataObject) { + this.dataObject = dataObject; + } + + public EndPoint getEndPoint() { + return endPoint; + } + + public void setEndPoint(EndPoint endPoint) { + this.endPoint = endPoint; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand) o; + + return dataObject.equals(that.dataObject) && endPoint.equals(that.endPoint); + + } + + @Override + public int hashCode() { + int result = dataObject.hashCode(); + result = 31 * result + endPoint.hashCode(); + return result; + } + + @Override public String toString() { + return "TemplateOrVolumePostUploadCommand{" + + "dataObject=" + dataObject + + ", endPoint=" + endPoint + + '}'; + } +} diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index 1be3439570d2..8363309a6e08 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -41,6 +41,7 @@ import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.log4j.Logger; @@ -104,6 +105,12 @@ public VMTemplateVO create(TemplateProfile profile) { return template; } + @Override + public List createTemplateForPostUpload(TemplateProfile profile) { + // TODO: support baremetal for postupload + return null; + } + @Override public TemplateProfile prepareDelete(DeleteIsoCmd cmd) { throw new CloudRuntimeException("Baremetal doesn't support ISO, how the delete get here???"); diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 42f4b554caed..8a9091585b69 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -25,6 +26,10 @@ import javax.ejb.Local; import javax.inject.Inject; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplate; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.log4j.Logger; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; @@ -131,6 +136,15 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio return profile; } + @Override + public TemplateProfile prepare(GetUploadParamsForTemplate cmd) throws ResourceAllocationException { + TemplateProfile profile = super.prepare(cmd); + + // Check that the resource limit for secondary storage won't be exceeded + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage); + return profile; + } + @Override public VMTemplateVO create(TemplateProfile profile) { // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync. @@ -185,6 +199,70 @@ public VMTemplateVO create(TemplateProfile profile) { return template; } + @Override + public List createTemplateForPostUpload(TemplateProfile profile) { + // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync. + VMTemplateVO template = persistTemplate(profile); + + if (template == null) { + throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate()); + } + + // find all eligible image stores for this zone scope + List imageStores = storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId())); + if (imageStores == null || imageStores.size() == 0) { + throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + } + + List payloads = new LinkedList<>(); + Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. + for (DataStore imageStore : imageStores) { + // skip data stores for a disabled zone + Long zoneId = imageStore.getScope().getScopeId(); + if (zoneId != null) { + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId()); + continue; + } + + // Check if zone is disabled + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + s_logger.info("Zone " + zoneId + " is disabled, so skip downloading template to its image store " + imageStore.getId()); + continue; + } + + // We want to download private template to one of the image store in a zone + if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) { + continue; + } else { + zoneSet.add(zoneId); + } + + } + + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); + //imageService.createTemplateAsync(tmpl, imageStore, caller); + + // persist template_store_ref entry + DataObject templateOnStore = imageStore.create(tmpl); + // update template_store_ref and template state + + EndPoint ep = _epSelector.select(templateOnStore); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + + TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(templateOnStore, ep); + payloads.add(payload); + } + _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); + return payloads; + } + private boolean isPrivateTemplate(VMTemplateVO template){ // if public OR featured OR system template diff --git a/server/src/com/cloud/template/TemplateAdapter.java b/server/src/com/cloud/template/TemplateAdapter.java index a85e3379834d..fb1e841cc140 100755 --- a/server/src/com/cloud/template/TemplateAdapter.java +++ b/server/src/com/cloud/template/TemplateAdapter.java @@ -16,12 +16,14 @@ // under the License. package com.cloud.template; +import java.util.List; import java.util.Map; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplate; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import com.cloud.exception.ResourceAllocationException; @@ -31,6 +33,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; import com.cloud.utils.component.Adapter; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; public interface TemplateAdapter extends Adapter { public static class TemplateAdapterType { @@ -50,10 +53,14 @@ public String getName() { public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocationException; + public TemplateProfile prepare(GetUploadParamsForTemplate cmd) throws ResourceAllocationException; + public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException; public VMTemplateVO create(TemplateProfile profile); + public List createTemplateForPostUpload(TemplateProfile profile); + public TemplateProfile prepareDelete(DeleteTemplateCmd cmd); public TemplateProfile prepareDelete(DeleteIsoCmd cmd); diff --git a/server/src/com/cloud/template/TemplateAdapterBase.java b/server/src/com/cloud/template/TemplateAdapterBase.java index 92383a031edb..a313bdd8c152 100755 --- a/server/src/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/com/cloud/template/TemplateAdapterBase.java @@ -22,6 +22,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplate; import org.apache.log4j.Logger; import org.apache.cloudstack.api.ApiConstants; @@ -182,9 +183,10 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d throw new InvalidParameterValueException("Please specify a valid zone Id. Only admins can create templates in all zones."); } - if (url.toLowerCase().contains("file://")) { - throw new InvalidParameterValueException("File:// type urls are currently unsupported"); - } + //TODO: commenting it for post upload to work as it doesnt have an url. Need to figure out how to resolve this +// if (url.toLowerCase().contains("file://")) { +// throw new InvalidParameterValueException("File:// type urls are currently unsupported"); +// } // check whether owner can create public templates boolean allowPublicUserTemplates = TemplateManager.AllowPublicUserTemplates.valueIn(templateOwner.getId()); @@ -272,6 +274,29 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio } + @Override + public TemplateProfile prepare(GetUploadParamsForTemplate cmd) throws ResourceAllocationException { + //check if the caller can operate with the template owner + Account caller = CallContext.current().getCallingAccount(); + Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); + _accountMgr.checkAccess(caller, null, true, owner); + + boolean isRouting = (cmd.isRoutingType() == null) ? false : cmd.isRoutingType(); + + Long zoneId = cmd.getZoneId(); + // ignore passed zoneId if we are using region wide image store + List stores = _imgStoreDao.findRegionImageStores(); + if (stores != null && stores.size() > 0) { + zoneId = -1L; + } + + return prepare(false, CallContext.current().getCallingUserId(), cmd.getName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), + cmd.getRequiresHvm(), null, cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, + HypervisorType.getType(cmd.getHypervisor()), cmd.getChecksum(), true, cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, + cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER); + + } + @Override public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException { //check if the caller can operate with the template owner diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index aac040b8feff..18fe28bc4a53 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -16,20 +16,33 @@ // under the License. package com.cloud.template; +import java.net.MalformedURLException; import java.net.URISyntaxException; +import java.net.URL; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.google.gson.Gson; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplate; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -178,6 +191,8 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; @Local(value = {TemplateManager.class, TemplateApiService.class}) public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiService, Configurable { @@ -319,6 +334,70 @@ public VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws U } } + @Override + @ActionEvent(eventType = EventTypes.EVENT_TEMPLATE_CREATE, eventDescription = "creating post upload template") + public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplate cmd) throws ResourceAllocationException, MalformedURLException { + TemplateAdapter adapter = getAdapter(HypervisorType.getType(cmd.getHypervisor())); + TemplateProfile profile = adapter.prepare(cmd); + List payload = adapter.createTemplateForPostUpload(profile); + + if(CollectionUtils.isNotEmpty(payload)) { + GetUploadParamsResponse response = new GetUploadParamsResponse(); + + /* + * There can be one or more commands depending on the number of secondary stores the template needs to go to. Taking the first one to do the url upload. The + * template will be propagated to the rest through copy. + */ + TemplateOrVolumePostUploadCommand firstCommand = payload.get(0); + + String url = "https://" + firstCommand.getEndPoint().getPublicAddr() + "/upload/" + firstCommand.getDataObject().getUuid(); + response.setPostURL(new URL(url)); + + response.setId(UUID.fromString(firstCommand.getDataObject().getUuid())); + + /* + * TODO: hardcoding the timeout to current + 60 min for now. This needs to goto the database + */ + DateTime currentDateTime = new DateTime(DateTimeZone.UTC); + currentDateTime.plusHours(1); + String expires = currentDateTime.toString(); + response.setTimeout(expires); + + /* + * encoded metadata using the post upload config ssh key + */ + Gson gson = new Gson(); + String jsonPayload = gson.toJson(payload); + response.setMetadata(encodeData(jsonPayload)); + + /* + * signature calculated on the url, expiry, metadata. + */ + response.setSignature(encodeData(jsonPayload+url+expires)); + + return response; + } else { + return null; + } + } + + private String encodeData(String data) { + String key = _configDao.getValue(Config.SSVMPSK.key()); + + try { + final Mac mac = Mac.getInstance("HmacSHA1"); + final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1"); + mac.init(keySpec); + mac.update(data.getBytes()); + final byte[] encryptedBytes = mac.doFinal(); + final String computedSignature = Base64.encodeBase64String(encryptedBytes); + return computedSignature; + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + s_logger.error("exception occured which encoding the data.", e); + return null; + } + } + @Override public DataStore getImageStore(String storeUuid, Long zoneId) { DataStore imageStore = null;