diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index e7c5efb773b1..e2ea408e7b8c 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -333,6 +333,8 @@ public boolean isUsedBySystem() { */ Date getCreated(); + Date getRemoved(); + long getServiceOfferingId(); Long getBackupOfferingId(); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 55046e8bd3d6..21da4025e846 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1145,6 +1145,7 @@ public class ApiConstants { public static final String WEBHOOK_NAME = "webhookname"; public static final String NFS_MOUNT_OPTIONS = "nfsmountopts"; + public static final String MOUNT_OPTIONS = "mountopts"; public static final String SHAREDFSVM_MIN_CPU_COUNT = "sharedfsvmmincpucount"; public static final String SHAREDFSVM_MIN_RAM_SIZE = "sharedfsvmminramsize"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 871b973254c2..46a810c6f3b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,8 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.backup.BackupRepository; import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -554,5 +556,7 @@ List createTemplateResponses(ResponseView view, VirtualMachine BucketResponse createBucketResponse(Bucket bucket); + BackupRepositoryResponse createBackupRepositoryResponse(BackupRepository repository); + SharedFSResponse createSharedFSResponse(ResponseView view, SharedFS sharedFS); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java index 6cc765328f61..a76107174358 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java @@ -19,6 +19,7 @@ import javax.inject.Inject; +import com.amazonaws.util.CollectionUtils; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -27,6 +28,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupSchedule; @@ -39,6 +41,9 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.exception.CloudRuntimeException; +import java.util.ArrayList; +import java.util.List; + @APICommand(name = "listBackupSchedule", description = "List backup schedule of a VM", responseObject = BackupScheduleResponse.class, since = "4.14.0", @@ -74,9 +79,14 @@ public Long getVmId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try{ - BackupSchedule schedule = backupManager.listBackupSchedule(getVmId()); - if (schedule != null) { - BackupScheduleResponse response = _responseGenerator.createBackupScheduleResponse(schedule); + List schedules = backupManager.listBackupSchedule(getVmId()); + ListResponse response = new ListResponse<>(); + List scheduleResponses = new ArrayList<>(); + if (CollectionUtils.isNullOrEmpty(schedules)) { + for (BackupSchedule schedule : schedules) { + scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule)); + } + response.setResponses(scheduleResponses, schedules.size()); response.setResponseName(getCommandName()); setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java new file mode 100644 index 000000000000..5d0c838bc377 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryService; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; + +@APICommand(name = "addBackupRepository", + description = "Adds a backup repository to store NAS backups", + responseObject = BackupRepositoryResponse.class, since = "4.20.0", + authorized = {RoleType.Admin}) +public class AddBackupRepositoryCmd extends BaseCmd { + + @Inject + private BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "name of the backup repository") + private String name; + + @Parameter(name = ApiConstants.ADDRESS, type = CommandType.STRING, required = true, description = "address of the backup repository") + private String address; + + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, required = true, description = "type of the backup repository storage. Supported values: nfs, cephfs, cifs") + private String type; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "backup repository provider") + private String provider; + + @Parameter(name = ApiConstants.MOUNT_OPTIONS, type = CommandType.STRING, description = "shared storage mount options") + private String mountOptions; + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + required = true, + description = "ID of the zone where the backup repository is to be added") + private Long zoneId; + + @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, description = "capacity of this backup repository") + private Long capacityBytes; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public BackupRepositoryService getBackupRepositoryService() { + return backupRepositoryService; + } + + public String getName() { + return name; + } + + public String getType() { + if ("cephfs".equalsIgnoreCase(type)) { + return "ceph"; + } + return type.toLowerCase(); + } + + public String getAddress() { + return address; + } + + public String getProvider() { + return provider; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public Long getZoneId() { + return zoneId; + } + + public Long getCapacityBytes() { + return capacityBytes; + } + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + try { + BackupRepository result = backupRepositoryService.addBackupRepository(this); + if (result != null) { + BackupRepositoryResponse response = _responseGenerator.createBackupRepositoryResponse(result); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add backup repository"); + } + } catch (Exception ex4) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex4.getMessage()); + } + + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/DeleteBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/DeleteBackupRepositoryCmd.java new file mode 100644 index 000000000000..912170eb4ca2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/DeleteBackupRepositoryCmd.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.BackupRepositoryService; + +import javax.inject.Inject; + +@APICommand(name = "deleteBackupRepository", + description = "delete a backup repository", + responseObject = SuccessResponse.class, since = "4.20.0", + authorized = {RoleType.Admin}) +public class DeleteBackupRepositoryCmd extends BaseCmd { + + @Inject + BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupRepositoryResponse.class, + required = true, + description = "ID of the backup repository to be deleted") + private Long id; + + + ///////////////////////////////////////////////////// + //////////////// Accessors ////////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public void execute() { + boolean result = backupRepositoryService.deleteBackupRepository(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete backup repository"); + } + } + + @Override + public long getEntityOwnerId() { + return 0; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/ListBackupRepositoriesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/ListBackupRepositoriesCmd.java new file mode 100644 index 000000000000..8293afb657d5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/ListBackupRepositoriesCmd.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.Pair; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryService; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listBackupRepositories", + description = "Lists all backup repositories", + responseObject = BackupRepositoryResponse.class, since = "4.20.0", + authorized = {RoleType.Admin}) +public class ListBackupRepositoriesCmd extends BaseListCmd { + + @Inject + BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the backup repository") + private String name; + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "ID of the zone where the backup repository is to be added") + private Long zoneId; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "the backup repository provider") + private String provider; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = BackupRepositoryResponse.class, description = "ID of the backup repository") + private Long id; + + ///////////////////////////////////////////////////// + //////////////// Accessors ////////////////////////// + ///////////////////////////////////////////////////// + + + public String getName() { + return name; + } + + public Long getZoneId() { + return zoneId; + } + + public String getProvider() { + return provider; + } + + public Long getId() { + return id; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + Pair, Integer> repositoriesPair = backupRepositoryService.listBackupRepositories(this); + List backupRepositories = repositoriesPair.first(); + ListResponse response = new ListResponse<>(); + List responses = new ArrayList<>(); + for (BackupRepository repository : backupRepositories) { + responses.add(_responseGenerator.createBackupRepositoryResponse(repository)); + } + response.setResponses(responses, repositoriesPair.second()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + String msg = String.format("Error listing backup repositories, due to: %s", e.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java new file mode 100644 index 000000000000..3847176608c0 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.backup.BackupRepository; + +import java.util.Date; + +@EntityReference(value = BackupRepository.class) +public class BackupRepositoryResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the backup repository") + private String id; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "the Zone ID of the backup repository") + private String zoneId; + + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "the Zone name of the backup repository") + private String zoneName; + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the backup repository") + private String name; + + @SerializedName(ApiConstants.ADDRESS) + @Param(description = "the address / url of the backup repository") + private String address; + + @SerializedName(ApiConstants.PROVIDER) + @Param(description = "name of the provider") + private String providerName; + + @SerializedName(ApiConstants.TYPE) + @Param(description = "backup type") + private String type; + + @SerializedName(ApiConstants.MOUNT_OPTIONS) + @Param(description = "mount options for the backup repository") + private String mountOptions; + + @SerializedName(ApiConstants.CAPACITY_BYTES) + @Param(description = "capacity of the backup repository") + private Long capacityBytes; + + @SerializedName("created") + @Param(description = "the date and time the backup repository was added") + private Date created; + + public BackupRepositoryResponse() { + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getZoneName() { + return zoneName; + } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + public String getProviderName() { + return providerName; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Long getCapacityBytes() { + return capacityBytes; + } + + public void setCapacityBytes(Long capacityBytes) { + this.capacityBytes = capacityBytes; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index df1b243dbabb..f21f20adb33e 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.backup; import java.util.Date; +import java.util.List; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Identity; @@ -141,5 +142,6 @@ public String toString() { Backup.Status getStatus(); Long getSize(); Long getProtectedSize(); + List getBackedUpVolumes(); long getZoneId(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 7b39804c738e..8b45bb4ee5ef 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -107,7 +107,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param vmId * @return */ - BackupSchedule listBackupSchedule(Long vmId); + List listBackupSchedule(Long vmId); /** * Deletes VM backup schedule for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 9c1b14ae60f2..d36dfb7360f6 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -93,7 +93,7 @@ public interface BackupProvider { /** * Restore a volume from a backup */ - Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid); + Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState); /** * Returns backup metrics for a list of VMs in a zone diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java new file mode 100644 index 000000000000..8e5c9740e690 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -0,0 +1,34 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface BackupRepository extends InternalIdentity, Identity { + String getProvider(); + long getZoneId(); + String getName(); + String getType(); + String getAddress(); + String getMountOptions(); + Long getCapacityBytes(); + Long getUsedBytes(); + Date getCreated(); +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java new file mode 100644 index 000000000000..ae71053e400d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.utils.Pair; +import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; + +import java.util.List; + +public interface BackupRepositoryService { + BackupRepository addBackupRepository(AddBackupRepositoryCmd cmd); + boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd); + Pair, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd); + +} diff --git a/client/pom.xml b/client/pom.xml index 473b711b9e32..03f6520cb05a 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -628,6 +628,11 @@ cloud-plugin-backup-networker ${project.version} + + org.apache.cloudstack + cloud-plugin-backup-nas + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java new file mode 100644 index 000000000000..09f9c5621502 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -0,0 +1,59 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; + +import java.util.Map; + +public class BackupAnswer extends Answer { + private Long size; + private Long virtualSize; + private Map volumes; + + public BackupAnswer(final Command command, final boolean success, final String details) { + super(command, success, details); + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getVirtualSize() { + return virtualSize; + } + + public void setVirtualSize(Long virtualSize) { + this.virtualSize = virtualSize; + } + + public Map getVolumes() { + return volumes; + } + + public void setVolumes(Map volumes) { + this.volumes = volumes; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java new file mode 100644 index 000000000000..16c611af998e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java @@ -0,0 +1,76 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class DeleteBackupCommand extends Command { + private String backupPath; + private String backupRepoType; + private String backupRepoAddress; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public DeleteBackupCommand(String backupPath, String backupRepoType, String backupRepoAddress, String mountOptions) { + super(); + this.backupPath = backupPath; + this.backupRepoType = backupRepoType; + this.backupRepoAddress = backupRepoAddress; + this.mountOptions = mountOptions; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java new file mode 100644 index 000000000000..7228e35147af --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -0,0 +1,130 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; +import com.cloud.vm.VirtualMachine; + +import java.util.List; + +public class RestoreBackupCommand extends Command { + private String vmName; + private String backupPath; + private String backupRepoType; + private String backupRepoAddress; + private List volumePaths; + private String diskType; + private Boolean vmExists; + private String restoreVolumeUUID; + private VirtualMachine.State vmState; + + protected RestoreBackupCommand() { + super(); + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public List getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(List volumePaths) { + this.volumePaths = volumePaths; + } + + public Boolean isVmExists() { + return vmExists; + } + + public void setVmExists(Boolean vmExists) { + this.vmExists = vmExists; + } + + public String getDiskType() { + return diskType; + } + + public void setDiskType(String diskType) { + this.diskType = diskType; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + public String getRestoreVolumeUUID() { + return restoreVolumeUUID; + } + + public void setRestoreVolumeUUID(String restoreVolumeUUID) { + this.restoreVolumeUUID = restoreVolumeUUID; + } + + public VirtualMachine.State getVmState() { + return vmState; + } + + public void setVmState(VirtualMachine.State vmState) { + this.vmState = vmState; + } + + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + @Override + + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java new file mode 100644 index 000000000000..93855ea17211 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -0,0 +1,94 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +import java.util.List; + +public class TakeBackupCommand extends Command { + private String vmName; + private String backupPath; + private String backupRepoType; + private String backupRepoAddress; + private List volumePaths; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public TakeBackupCommand(String vmName, String backupPath) { + super(); + this.vmName = vmName; + this.backupPath = backupPath; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + public List getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(List volumePaths) { + this.volumePaths = volumePaths; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/debian/control b/debian/control index ce136ea983a7..8a098e97165f 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor, cpu-checker +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java new file mode 100644 index 000000000000..e8364520ed05 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.utils.db.Encrypt; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "backup_repository") +public class BackupRepositoryVO implements BackupRepository { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "zone_id", nullable = false) + private long zoneId; + + @Column(name = "provider", nullable = false) + private String provider; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "address", nullable = false) + private String address; + + @Encrypt + @Column(name = "mount_opts") + private String mountOptions; + + @Column(name = "used_bytes",nullable = true) + private Long usedBytes; + + @Column(name = "capacity_bytes", nullable = true) + private Long capacityBytes; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public BackupRepositoryVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes) { + this(); + this.zoneId = zoneId; + this.provider = provider; + this.name = name; + this.type = type; + this.address = address; + this.mountOptions = mountOptions; + this.capacityBytes = capacityBytes; + this.created = new Date(); + } + + public String getUuid() { + return uuid; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public long getZoneId() { + return zoneId; + } + + @Override + public String getProvider() { + return provider; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public void setAddress(String address) { + this.address = address; + } + + @Override + public String getAddress() { + return address; + } + + @Override + public String getMountOptions() { + return mountOptions; + } + + @Override + public Long getUsedBytes() { + return usedBytes; + } + + @Override + public Long getCapacityBytes() { + return capacityBytes; + } + + public Date getCreated() { + return created; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index b6ad0e7cb8fa..9b285e66cab9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -18,8 +18,13 @@ package org.apache.cloudstack.backup; import com.cloud.utils.db.GenericDao; +import com.google.gson.Gson; +import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.UUID; import javax.persistence.Column; @@ -82,6 +87,9 @@ public class BackupVO implements Backup { @Column(name = "zone_id") private long zoneId; + @Column(name = "backed_volumes", length = 65535) + protected String backedUpVolumes; + public BackupVO() { this.uuid = UUID.randomUUID().toString(); } @@ -203,6 +211,17 @@ public String getName() { return null; } + public List getBackedUpVolumes() { + if (StringUtils.isEmpty(this.backedUpVolumes)) { + return Collections.emptyList(); + } + return Arrays.asList(new Gson().fromJson(this.backedUpVolumes, Backup.VolumeInfo[].class)); + } + + public void setBackedUpVolumes(String backedUpVolumes) { + this.backedUpVolumes = backedUpVolumes; + } + public Date getRemoved() { return removed; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 5d2f5ac64d61..89a13245b0a0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -32,9 +32,8 @@ public interface BackupDao extends GenericDao { List listByVmId(Long zoneId, Long vmId); List listByAccountId(Long accountId); - List listByOfferingId(Long offeringId); List syncBackups(Long zoneId, Long vmId, List externalBackups); BackupVO getBackupVO(Backup backup); - + List listByOfferingId(Long backupOfferingId); BackupResponse newBackupResponse(Backup backup); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 8628fe8e01b2..5a9cd0620374 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import javax.annotation.PostConstruct; import javax.inject.Inject; @@ -68,6 +69,8 @@ protected void init() { backupSearch = createSearchBuilder(); backupSearch.and("vm_id", backupSearch.entity().getVmId(), SearchCriteria.Op.EQ); backupSearch.and("external_id", backupSearch.entity().getExternalId(), SearchCriteria.Op.EQ); + backupSearch.and("backup_offering_id", backupSearch.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); + backupSearch.and("zone_id", backupSearch.entity().getZoneId(), SearchCriteria.Op.EQ); backupSearch.done(); } @@ -102,13 +105,6 @@ public List listByVmId(Long zoneId, Long vmId) { return new ArrayList<>(listBy(sc)); } - @Override - public List listByOfferingId(Long offeringId) { - SearchCriteria sc = backupSearch.create(); - sc.setParameters("offering_id", offeringId); - return new ArrayList<>(listBy(sc)); - } - private Backup findByExternalId(Long zoneId, String externalId) { SearchCriteria sc = backupSearch.create(); sc.setParameters("external_id", externalId); @@ -123,6 +119,13 @@ public BackupVO getBackupVO(Backup backup) { return backupVO; } + @Override + public List listByOfferingId(Long backupOfferingId) { + SearchCriteria sc = backupSearch.create(); + sc.setParameters("backup_offering_id", backupOfferingId); + return new ArrayList<>(listBy(sc)); + } + public void removeExistingBackups(Long zoneId, Long vmId) { SearchCriteria sc = backupSearch.create(); sc.setParameters("vm_id", vmId); @@ -145,9 +148,9 @@ public BackupResponse newBackupResponse(Backup backup) { AccountVO account = accountDao.findByIdIncludingRemoved(vm.getAccountId()); DomainVO domain = domainDao.findByIdIncludingRemoved(vm.getDomainId()); DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(vm.getDataCenterId()); - Long offeringId = vm.getBackupOfferingId(); + Long offeringId = backup.getBackupOfferingId(); if (offeringId == null) { - offeringId = backup.getBackupOfferingId(); + offeringId = vm.getBackupOfferingId(); } BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(offeringId); @@ -161,7 +164,14 @@ public BackupResponse newBackupResponse(Backup backup) { response.setSize(backup.getSize()); response.setProtectedSize(backup.getProtectedSize()); response.setStatus(backup.getStatus()); - response.setVolumes(new Gson().toJson(vm.getBackupVolumeList().toArray(), Backup.VolumeInfo[].class)); + // ACS 4.20: For backups taken prior this release the backup.backed_volumes column would be empty hence use vm_instance.backup_volumes + String backedUpVolumes; + if (Objects.isNull(backup.getBackedUpVolumes())) { + backedUpVolumes = new Gson().toJson(vm.getBackupVolumeList().toArray(), Backup.VolumeInfo[].class); + } else { + backedUpVolumes = new Gson().toJson(backup.getBackedUpVolumes().toArray(), Backup.VolumeInfo[].class); + } + response.setVolumes(backedUpVolumes); response.setBackupOfferingId(offering.getUuid()); response.setBackupOffering(offering.getName()); response.setAccountId(account.getUuid()); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java new file mode 100644 index 000000000000..0034bfb30ab6 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryVO; + +import com.cloud.utils.db.GenericDao; + +public interface BackupRepositoryDao extends GenericDao { + List listByZoneAndProvider(Long zoneId, String provider); + + BackupRepository findByBackupOfferingId(Long backupOfferingId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java new file mode 100644 index 000000000000..460b6d8aba45 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.cloudstack.backup.BackupOfferingVO; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryVO; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class BackupRepositoryDaoImpl extends GenericDaoBase implements BackupRepositoryDao { + @Inject + BackupOfferingDao backupOfferingDao; + + private SearchBuilder backupRepoSearch; + + public BackupRepositoryDaoImpl() { + } + + @PostConstruct + protected void init() { + backupRepoSearch = createSearchBuilder(); + backupRepoSearch.and("zone_id", backupRepoSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + backupRepoSearch.and("provider", backupRepoSearch.entity().getProvider(), SearchCriteria.Op.EQ); + backupRepoSearch.done(); + } + + @Override + public List listByZoneAndProvider(Long zoneId, String provider) { + SearchCriteria sc = backupRepoSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("provider", provider); + return new ArrayList<>(listBy(sc)); + } + + @Override + public BackupRepository findByBackupOfferingId(Long backupOfferingId) { + BackupOfferingVO offering = backupOfferingDao.findByIdIncludingRemoved(backupOfferingId); + if (offering == null) { + return null; + } + return findByUuid(offering.getExternalId()); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java index 516b0112c986..ee1783a9c896 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java @@ -20,6 +20,7 @@ import java.util.Date; import java.util.List; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.BackupScheduleVO; @@ -29,6 +30,10 @@ public interface BackupScheduleDao extends GenericDao { BackupScheduleVO findByVM(Long vmId); + List listByVM(Long vmId); + + BackupScheduleVO findByVMAndIntervalType(Long vmId, DateUtil.IntervalType intervalType); + List getSchedulesToExecute(Date currentTimestamp); BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index 7a58679e7e53..e00ccc5abd77 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -23,6 +23,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.BackupScheduleVO; @@ -49,6 +50,7 @@ protected void init() { backupScheduleSearch = createSearchBuilder(); backupScheduleSearch.and("vm_id", backupScheduleSearch.entity().getVmId(), SearchCriteria.Op.EQ); backupScheduleSearch.and("async_job_id", backupScheduleSearch.entity().getAsyncJobId(), SearchCriteria.Op.EQ); + backupScheduleSearch.and("interval_type", backupScheduleSearch.entity().getScheduleType(), SearchCriteria.Op.EQ); backupScheduleSearch.done(); executableSchedulesSearch = createSearchBuilder(); @@ -64,6 +66,21 @@ public BackupScheduleVO findByVM(Long vmId) { return findOneBy(sc); } + @Override + public List listByVM(Long vmId) { + SearchCriteria sc = backupScheduleSearch.create(); + sc.setParameters("vm_id", vmId); + return listBy(sc, null); + } + + @Override + public BackupScheduleVO findByVMAndIntervalType(Long vmId, DateUtil.IntervalType intervalType) { + SearchCriteria sc = backupScheduleSearch.create(); + sc.setParameters("vm_id", vmId); + sc.setParameters("interval_type", intervalType.ordinal()); + return findOneBy(sc); + } + @Override public List getSchedulesToExecute(Date currentTimestamp) { SearchCriteria sc = executableSchedulesSearch.create(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index aee58be99b7b..0be1e6100098 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -269,6 +269,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql index 3dff78d54917..041997557aa6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -457,6 +457,35 @@ CALL `cloud`.`IDEMPOTENT_MODIFY_COLUMN_CHAR_SET`('vpc_offerings', 'display_text' CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.roles','state', 'varchar(10) NOT NULL default "enabled" COMMENT "role state"'); +-- NAS B&R Plugin Backup Repository +DROP TABLE IF EXISTS `cloud`.`backup_repository`; +CREATE TABLE `cloud`.`backup_repository` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id of the backup repository', + `uuid` varchar(255) NOT NULL COMMENT 'uuid of the backup repository', + `name` varchar(255) CHARACTER SET utf8mb4 NOT NULL COMMENT 'name of the backup repository', + `zone_id` bigint unsigned NOT NULL COMMENT 'id of zone', + `provider` varchar(255) NOT NULL COMMENT 'backup provider name', + `type` varchar(255) NOT NULL COMMENT 'backup repo type', + `address` varchar(1024) NOT NULL COMMENT 'url of the backup repository', + `mount_opts` varchar(1024) NOT NULL COMMENT 'mount options for the backup repository', + `used_bytes` bigint unsigned, + `capacity_bytes` bigint unsigned, + `created` datetime, + `removed` datetime, + PRIMARY KEY(`id`), + INDEX `i_backup_repository__uuid`(`uuid`), + INDEX `i_backup_repository__zone_id_provider`(`zone_id`, `provider`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Drop foreign key on backup_schedule, drop unique key on vm_id and re-add foreign key to allow multiple backup schedules to be created +ALTER TABLE `cloud`.`backup_schedule` DROP FOREIGN KEY fk_backup_schedule__vm_id; +ALTER TABLE `cloud`.`backup_schedule` DROP INDEX vm_id; +ALTER TABLE `cloud`.`backup_schedule` ADD CONSTRAINT fk_backup_schedule__vm_id FOREIGN KEY (vm_id) REFERENCES vm_instance(id) ON DELETE CASCADE; + +-- Add volume details to the backups table to keep track of the volumes being backed up +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'backed_volumes', 'text DEFAULT NULL COMMENT "details of backed-up volumes" '); +CALL `cloud`.`IDEMPOTENT_MODIFY_COLUMN_CHAR_SET`('backups', 'backed_volumes', 'TEXT', 'DEFAULT NULL COMMENT \'details of backed-up volumes\''); + -- Add support for VMware 8.0u2 (8.0.2.x) and 8.0u3 (8.0.3.x) INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.2', 1024, 0, 59, 64, 1, 1); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '8.0.2', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='8.0'; diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index adc8c2e2b35e..a88d4b1cbbf9 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -109,6 +109,7 @@ Requires: (net-tools or net-tools-deprecated) Requires: iproute Requires: ipset Requires: perl +Requires: rsync Requires: (python3-libvirt or python3-libvirt-python) Requires: (qemu-img or qemu-tools) Requires: qemu-kvm diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index fa376f992ed1..f162c51a703d 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import com.cloud.storage.dao.VolumeDao; import org.apache.cloudstack.backup.dao.BackupDao; import com.cloud.utils.Pair; @@ -37,6 +38,8 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { @Inject private BackupDao backupDao; + @Inject + private VolumeDao volumeDao; @Override public String getName() { @@ -76,7 +79,7 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid) { + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { logger.debug("Restoring volume " + volumeUuid + "from backup " + backup.getUuid() + " on the Dummy Backup Provider"); throw new CloudRuntimeException("Dummy plugin does not support this feature"); } @@ -123,6 +126,7 @@ public boolean takeBackup(VirtualMachine vm) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); return backupDao.persist(backup) != null; } diff --git a/plugins/backup/nas/pom.xml b/plugins/backup/nas/pom.xml new file mode 100644 index 000000000000..096bf45c67ed --- /dev/null +++ b/plugins/backup/nas/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + cloud-plugin-backup-nas + Apache CloudStack Plugin - KVM NAS Backup and Recovery Plugin + + cloudstack-plugins + org.apache.cloudstack + 4.20.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-plugin-hypervisor-kvm + ${project.version} + + + org.apache.commons + commons-lang3 + ${cs.commons-lang3.version} + + + com.fasterxml.jackson.core + jackson-databind + ${cs.jackson.version} + + + com.github.tomakehurst + wiremock-standalone + ${cs.wiremock.version} + test + + + diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java new file mode 100644 index 000000000000..4a6725abdca5 --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -0,0 +1,442 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import com.cloud.agent.AgentManager; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.ScopeType; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import javax.inject.Inject; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.HashMap; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { + private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + + @Inject + private BackupDao backupDao; + + @Inject + private BackupRepositoryDao backupRepositoryDao; + + @Inject + private BackupOfferingDao backupOfferingDao; + + @Inject + private HostDao hostDao; + + @Inject + private ClusterDao clusterDao; + + @Inject + private VolumeDao volumeDao; + + @Inject + private StoragePoolHostDao storagePoolHostDao; + + @Inject + private VMInstanceDao vmInstanceDao; + + @Inject + private PrimaryDataStoreDao primaryDataStoreDao; + + @Inject + private AgentManager agentManager; + + protected Host getLastVMHypervisorHost(VirtualMachine vm) { + Long hostId = vm.getLastHostId(); + if (hostId == null) { + LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); + return null; + } + Host host = hostDao.findById(hostId); + + if (host.getStatus() == Status.Up) { + return host; + } else { + // Try to find any Up host in the same cluster + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(host.getClusterId())) { + if (hostInCluster.getStatus() == Status.Up) { + LOG.debug("Found Host " + hostInCluster.getName()); + return hostInCluster; + } + } + } + // Try to find any Host in the zone + for (final HostVO hostInZone : hostDao.listByDataCenterIdAndHypervisorType(host.getDataCenterId(), Hypervisor.HypervisorType.KVM)) { + if (hostInZone.getStatus() == Status.Up) { + LOG.debug("Found Host " + hostInZone.getName()); + return hostInZone; + } + } + return null; + } + + protected Host getVMHypervisorHost(VirtualMachine vm) { + Long hostId = vm.getHostId(); + if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { + throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for %s. Make sure the virtual machine is running", vm.getName())); + } + if (VirtualMachine.State.Stopped.equals(vm.getState())) { + hostId = vm.getLastHostId(); + } + if (hostId == null) { + throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for stopped VM: %s", vm)); + } + final Host host = hostDao.findById(hostId); + if (host == null || !Status.Up.equals(host.getStatus()) || !Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType())) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } + return host; + } + + @Override + public boolean takeBackup(final VirtualMachine vm) { + final Host host = getVMHypervisorHost(vm); + + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); + } + + final Date creationDate = new Date(); + final String backupPath = String.format("%s/%s", vm.getInstanceName(), + new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(creationDate)); + + BackupVO backupVO = createBackupObject(vm, backupPath); + TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath); + command.setBackupRepoType(backupRepository.getType()); + command.setBackupRepoAddress(backupRepository.getAddress()); + command.setMountOptions(backupRepository.getMountOptions()); + + if (VirtualMachine.State.Stopped.equals(vm.getState())) { + List vmVolumes = volumeDao.findByInstance(vm.getId()); + List volumePaths = getVolumePaths(vmVolumes); + command.setVolumePaths(volumePaths); + } + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } + + if (answer != null && answer.getResult()) { + backupVO.setDate(new Date()); + backupVO.setSize(answer.getSize()); + backupVO.setStatus(Backup.Status.BackedUp); + backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + return backupDao.update(backupVO.getId(), backupVO); + } else { + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + } + return Objects.nonNull(answer) && answer.getResult(); + } + + private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(backupPath); + backup.setType("FULL"); + backup.setDate(new Date()); + long virtualSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vm.getId())) { + if (Volume.State.Ready.equals(volume.getState())) { + virtualSize += volume.getSize(); + } + } + backup.setProtectedSize(Long.valueOf(virtualSize)); + backup.setStatus(Backup.Status.BackingUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + return backupDao.persist(backup); + } + + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + List backedVolumes = backup.getBackedUpVolumes(); + List volumes = backedVolumes.stream().map(volume -> volumeDao.findByUuid(volume.getUuid())).collect(Collectors.toList()); + + LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm.getUuid(), backup.getUuid()); + BackupRepository backupRepository = getBackupRepository(vm, backup); + + final Host host = getLastVMHypervisorHost(vm); + RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); + restoreCommand.setBackupPath(backup.getExternalId()); + restoreCommand.setBackupRepoType(backupRepository.getType()); + restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); + restoreCommand.setVmName(vm.getName()); + restoreCommand.setVolumePaths(getVolumePaths(volumes)); + restoreCommand.setVmExists(vm.getRemoved() == null); + restoreCommand.setVmState(vm.getState()); + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), restoreCommand); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } + return answer.getResult(); + } + + private List getVolumePaths(List volumes) { + List volumePaths = new ArrayList<>(); + for (VolumeVO volume : volumes) { + StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); + if (Objects.isNull(storagePool)) { + throw new CloudRuntimeException("Unable to find storage pool associated to the volume"); + } + String volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + if (ScopeType.HOST.equals(storagePool.getScope())) { + volumePathPrefix = storagePool.getPath(); + } + volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); + } + return volumePaths; + } + + @Override + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + final VolumeVO volume = volumeDao.findByUuid(volumeUuid); + final VirtualMachine backupSourceVm = vmInstanceDao.findById(backup.getVmId()); + final StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); + final HostVO hostVO = hostDao.findByIp(hostIp); + + Optional matchingVolume = getBackedUpVolumeInfo(backupSourceVm.getBackupVolumeList(), volumeUuid); + Long backedUpVolumeSize = matchingVolume.isPresent() ? matchingVolume.get().getSize() : 0L; + + LOG.debug("Restoring vm volume" + volumeUuid + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); + BackupRepository backupRepository = getBackupRepository(backupSourceVm, backup); + + VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), + backup.getDomainId(), backup.getAccountId(), 0, null, + backup.getSize(), null, null, null); + String volumeUUID = UUID.randomUUID().toString(); + restoredVolume.setName("RestoredVol-"+volume.getName()); + restoredVolume.setProvisioningType(volume.getProvisioningType()); + restoredVolume.setUpdated(new Date()); + restoredVolume.setUuid(volumeUUID); + restoredVolume.setRemoved(null); + restoredVolume.setDisplayVolume(true); + restoredVolume.setPoolId(dataStore.getPoolId()); + restoredVolume.setPath(restoredVolume.getUuid()); + restoredVolume.setState(Volume.State.Copying); + restoredVolume.setSize(backedUpVolumeSize); + restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); + + RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); + restoreCommand.setBackupPath(backup.getExternalId()); + restoreCommand.setBackupRepoType(backupRepository.getType()); + restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); + restoreCommand.setVmName(vmNameAndState.first()); + restoreCommand.setVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID))); + restoreCommand.setDiskType(volume.getVolumeType().name().toLowerCase(Locale.ROOT)); + restoreCommand.setVmExists(null); + restoreCommand.setVmState(vmNameAndState.second()); + restoreCommand.setRestoreVolumeUUID(volumeUuid); + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(hostVO.getId(), restoreCommand); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } + + if (answer.getResult()) { + try { + volumeDao.persist(restoredVolume); + } catch (Exception e) { + throw new CloudRuntimeException("Unable to create restored volume due to: " + e); + } + } + + return new Pair<>(answer.getResult(), answer.getDetails()); + } + + private BackupRepository getBackupRepository(VirtualMachine vm, Backup backup) { + BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); + final String errorMessage = "No valid backup repository found for the VM, please check the attached backup offering"; + if (backupRepository == null) { + logger.warn(errorMessage + "Re-attempting with the backup offering associated with the backup"); + } + backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException(errorMessage); + } + return backupRepository; + } + + private Optional getBackedUpVolumeInfo(List backedUpVolumes, String volumeUuid) { + return backedUpVolumes.stream() + .filter(v -> v.getUuid().equals(volumeUuid)) + .findFirst(); + } + + @Override + public boolean deleteBackup(Backup backup, boolean forced) { + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); + } + + final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + final Host host = getLastVMHypervisorHost(vm); + + DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), + backupRepository.getAddress(), backupRepository.getMountOptions()); + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } + + if (answer != null && answer.getResult()) { + return backupDao.remove(backup.getId()); + } + + return false; + } + + @Override + public Map getBackupMetrics(Long zoneId, List vms) { + final Map metrics = new HashMap<>(); + if (CollectionUtils.isEmpty(vms)) { + LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); + return metrics; + } + + for (final VirtualMachine vm : vms) { + Long vmBackupSize = 0L; + Long vmBackupProtectedSize = 0L; + for (final Backup backup: backupDao.listByVmId(null, vm.getId())) { + vmBackupSize += backup.getSize(); + vmBackupProtectedSize += backup.getProtectedSize(); + } + Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); + LOG.debug(String.format("Metrics for VM [uuid: %s, name: %s] is [backup size: %s, data size: %s].", vm.getUuid(), + vm.getInstanceName(), vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); + metrics.put(vm, vmBackupMetric); + } + return metrics; + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + return true; + } + + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { + return false; + } + + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + // TODO: check and sum/return backups metrics on per VM basis + } + + @Override + public List listBackupOfferings(Long zoneId) { + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + final List offerings = new ArrayList<>(); + for (final BackupRepository repository : repositories) { + offerings.add(new NasBackupOffering(repository.getName(), repository.getUuid())); + } + return offerings; + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + return true; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + }; + } + + @Override + public String getName() { + return "nas"; + } + + @Override + public String getDescription() { + return "NAS Backup Plugin"; + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } +} diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java new file mode 100644 index 000000000000..91df74166e58 --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import java.util.Date; + +public class NasBackupOffering implements BackupOffering { + + private String name; + private String uid; + + public NasBackupOffering(String name, String uid) { + this.name = name; + this.uid = uid; + } + + @Override + public String getExternalId() { + return uid; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "NAS Backup Offering (Repository)"; + } + + @Override + public long getZoneId() { + return -1; + } + + @Override + public boolean isUserDrivenBackupAllowed() { + return true; + } + + @Override + public String getProvider() { + return "nas"; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public String getUuid() { + return uid; + } + + @Override + public long getId() { + return -1; + } +} diff --git a/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties new file mode 100644 index 000000000000..2e101ef02314 --- /dev/null +++ b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=nas +parent=backup diff --git a/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml new file mode 100644 index 000000000000..635ca66fbdee --- /dev/null +++ b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml @@ -0,0 +1,26 @@ + + + + + + + diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index e375b42aeb5b..0e87ad338871 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -372,7 +372,7 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid) { + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { String networkerServer; VolumeVO volume = volumeDao.findByUuid(volumeUuid); VMInstanceVO backupSourceVm = vmInstanceDao.findById(backup.getVmId()); @@ -512,6 +512,7 @@ public boolean takeBackup(VirtualMachine vm) { LOG.info ("EMC Networker finished backup job for vm " + vm.getName() + " with saveset Time: " + saveTime); BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); if (backup != null) { + backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); backupDao.persist(backup); return true; } else { diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 0e4537390186..4750e3264aac 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -291,7 +291,7 @@ private void prepareForBackupRestoration(VirtualMachine vm) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid) { + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, hostIp, dataStoreUuid); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 5d9645092158..f3e379f2c842 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -326,6 +326,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String createTmplPath; private String heartBeatPath; private String vmActivityCheckPath; + private String nasBackupPath; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -714,6 +715,10 @@ public String getVmActivityCheckPath() { return vmActivityCheckPath; } + public String getNasBackupPath() { + return nasBackupPath; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -984,6 +989,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find kvmvmactivity.sh"); } + nasBackupPath = Script.findScript(kvmScriptsDir, "nasbackup.sh"); + if (nasBackupPath == null) { + throw new ConfigurationException("Unable to find nasbackup.sh"); + } + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java new file mode 100644 index 000000000000..4772d3b472cf --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java @@ -0,0 +1,63 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.DeleteBackupCommand; + +import java.util.ArrayList; +import java.util.List; + +@ResourceWrapper(handles = DeleteBackupCommand.class) +public class LibvirtDeleteBackupCommandWrapper extends CommandWrapper { + @Override + public Answer execute(DeleteBackupCommand command, LibvirtComputingResource libvirtComputingResource) { + final String backupPath = command.getBackupPath(); + final String backupRepoType = command.getBackupRepoType(); + final String backupRepoAddress = command.getBackupRepoAddress(); + final String mountOptions = command.getMountOptions(); + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), + "-o", "delete", + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", mountOptions, + "-p", backupPath + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + logger.debug(String.format("Backup delete result: %s , exit code: %s", result.second(), result.first())); + + if (result.first() != 0) { + logger.debug(String.format("Failed to delete VM backup: %s", result.second())); + return new BackupAnswer(command, false, result.second()); + } + return new BackupAnswer(command, true, null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java new file mode 100644 index 000000000000..23ead355096d --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -0,0 +1,203 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.RestoreBackupCommand; +import org.apache.commons.lang3.RandomStringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +@ResourceWrapper(handles = RestoreBackupCommand.class) +public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper { + private static final String BACKUP_TEMP_FILE_PREFIX = "csbackup"; + private static final String MOUNT_COMMAND = "sudo mount -t %s %s %s"; + private static final String UMOUNT_COMMAND = "sudo umount %s"; + private static final String FILE_PATH_PLACEHOLDER = "%s/%s"; + private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --cache none"; + private static final String CURRRENT_DEVICE = "virsh domblklist --domain %s | tail -n 3 | head -n 1 | awk '{print $1}'"; + private static final String RSYNC_COMMAND = "rsync -az %s %s"; + + @Override + public Answer execute(RestoreBackupCommand command, LibvirtComputingResource serverResource) { + String vmName = command.getVmName(); + String backupPath = command.getBackupPath(); + String backupRepoAddress = command.getBackupRepoAddress(); + String backupRepoType = command.getBackupRepoType(); + String mountOptions = command.getMountOptions(); + Boolean vmExists = command.isVmExists(); + String diskType = command.getDiskType(); + List volumePaths = command.getVolumePaths(); + String restoreVolumeUuid = command.getRestoreVolumeUUID(); + + String newVolumeId = null; + if (Objects.isNull(vmExists)) { + String volumePath = volumePaths.get(0); + int lastIndex = volumePath.lastIndexOf("/"); + newVolumeId = volumePath.substring(lastIndex + 1); + restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState())); + } else if (Boolean.TRUE.equals(vmExists)) { + restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions); + } else { + restoreVolumesOfDestroyedVMs(volumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + } + + return new BackupAnswer(command, true, newVolumeId); + } + + private void restoreVolumesOfExistingVM(List volumePaths, String backupPath, + String backupRepoType, String backupRepoAddress, String mountOptions) { + String diskType = "root"; + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType); + try { + for (int idx = 0; idx < volumePaths.size(); idx++) { + String volumePath = volumePaths.get(idx); + Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); + diskType = "datadisk"; + try { + replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); + } + } + } finally { + unmountBackupDirectory(mountDirectory); + deleteTemporaryDirectory(mountDirectory); + } + + } + + private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, + String backupRepoType, String backupRepoAddress, String mountOptions) { + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType); + String diskType = "root"; + try { + for (int i = 0; i < volumePaths.size(); i++) { + String volumePath = volumePaths.get(i); + Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); + diskType = "datadisk"; + try { + replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); + } + } + } finally { + unmountBackupDirectory(mountDirectory); + deleteTemporaryDirectory(mountDirectory); + } + } + + private void restoreVolume(String backupPath, String backupRepoType, String backupRepoAddress, String volumePath, + String diskType, String volumeUUID, Pair vmNameAndState) { + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType); + Pair bkpPathAndVolUuid; + try { + bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); + try { + replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); + if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { + if (!attachVolumeToVm(vmNameAndState.first(), volumePath)) { + throw new CloudRuntimeException(String.format("Failed to attach volume to VM: %s", vmNameAndState.first())); + } + } + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); + } + } catch (Exception e) { + throw new CloudRuntimeException("Failed to restore volume", e); + } finally { + unmountBackupDirectory(mountDirectory); + deleteTemporaryDirectory(mountDirectory); + } + } + + + private String mountBackupDirectory(String backupRepoAddress, String backupRepoType) { + String randomChars = RandomStringUtils.random(5, true, false); + String mountDirectory = String.format("%s.%s",BACKUP_TEMP_FILE_PREFIX , randomChars); + try { + mountDirectory = Files.createTempDirectory(mountDirectory).toString(); + String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); + Script.runSimpleBashScript(mount); + } catch (Exception e) { + throw new CloudRuntimeException(String.format("Failed to mount %s to %s", backupRepoType, backupRepoAddress), e); + } + return mountDirectory; + } + + private void unmountBackupDirectory(String backupDirectory) { + try { + String umountCmd = String.format(UMOUNT_COMMAND, backupDirectory); + Script.runSimpleBashScript(umountCmd); + } catch (Exception e) { + throw new CloudRuntimeException(String.format("Failed to unmount backup directory: %s", backupDirectory), e); + } + } + + private void deleteTemporaryDirectory(String backupDirectory) { + try { + Files.deleteIfExists(Paths.get(backupDirectory)); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Failed to delete backup directory: %s", backupDirectory), e); + } + } + + private Pair getBackupPath(String mountDirectory, String volumePath, String backupPath, String diskType, String volumeUuid) { + String bkpPath = String.format(FILE_PATH_PLACEHOLDER, mountDirectory, backupPath); + int lastIndex = volumePath.lastIndexOf(File.separator); + String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(lastIndex + 1) : volumeUuid; + String backupFileName = String.format("%s.%s.qcow2", diskType.toLowerCase(Locale.ROOT), volUuid); + bkpPath = String.format(FILE_PATH_PLACEHOLDER, bkpPath, backupFileName); + return new Pair<>(bkpPath, volUuid); + } + + private void replaceVolumeWithBackup(String volumePath, String backupPath) throws IOException { + Script.runSimpleBashScript(String.format(RSYNC_COMMAND, backupPath, volumePath)); + } + + private boolean attachVolumeToVm(String vmName, String volumePath) { + String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName); + int exitValue = Script.runSimpleBashScriptForExitValue(String.format(ATTACH_DISK_COMMAND, vmName, volumePath, deviceToAttachDiskTo)); + return exitValue == 0; + } + + private String getDeviceToAttachDisk(String vmName) { + String currentDevice = Script.runSimpleBashScript(String.format(CURRRENT_DEVICE, vmName)); + char lastChar = currentDevice.charAt(currentDevice.length() - 1); + char incrementedChar = (char) (lastChar + 1); + return currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java new file mode 100644 index 000000000000..3c0cc53bb73b --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -0,0 +1,84 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.amazonaws.util.CollectionUtils; +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.TakeBackupCommand; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +@ResourceWrapper(handles = TakeBackupCommand.class) +public class LibvirtTakeBackupCommandWrapper extends CommandWrapper { + @Override + public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvirtComputingResource) { + final String vmName = command.getVmName(); + final String backupPath = command.getBackupPath(); + final String backupRepoType = command.getBackupRepoType(); + final String backupRepoAddress = command.getBackupRepoAddress(); + final String mountOptions = command.getMountOptions(); + final List diskPaths = command.getVolumePaths(); + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), + "-o", "backup", + "-v", vmName, + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", Objects.nonNull(mountOptions) ? mountOptions : "", + "-p", backupPath, + "-d", (Objects.nonNull(diskPaths) && !diskPaths.isEmpty()) ? String.join(",", diskPaths) : "" + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + if (result.first() != 0) { + logger.debug("Failed to take VM backup: " + result.second()); + return new BackupAnswer(command, false, result.second().trim()); + } + + long backupSize = 0L; + if (CollectionUtils.isNullOrEmpty(diskPaths)) { + List outputLines = Arrays.asList(result.second().trim().split("\n")); + if (!outputLines.isEmpty()) { + backupSize = Long.parseLong(outputLines.get(outputLines.size() - 1).trim()); + } + } else { + String[] outputLines = result.second().trim().split("\n"); + for(String line : outputLines) { + backupSize = backupSize + Long.parseLong(line.split(" ")[0].trim()); + } + } + + BackupAnswer answer = new BackupAnswer(command, true, result.second().trim()); + answer.setSize(backupSize); + return answer; + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 92fe7951649b..923d31495f8e 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -62,6 +62,7 @@ backup/dummy backup/networker + backup/nas ca/root-ca diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh new file mode 100755 index 000000000000..5b264321bd8d --- /dev/null +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -0,0 +1,169 @@ +#!/usr/bin/bash +## Licensed to the Apache Software Foundation (ASF) under one +## or more contributor license agreements. See the NOTICE file +## distributed with this work for additional information +## regarding copyright ownership. The ASF licenses this file +## to you under the Apache License, Version 2.0 (the +## "License"); you may not use this file except in compliance +## with the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. + +set -e + +# CloudStack B&R NAS Backup and Recovery Tool for KVM + +# TODO: do libvirt/logging etc checks + +### Declare variables ### + +OP="" +VM="" +NAS_TYPE="" +NAS_ADDRESS="" +MOUNT_OPTS="" +BACKUP_DIR="" +DISK_PATHS="" + +### Operation methods ### + +backup_running_vm() { + mount_operation + mkdir -p $dest + + name="root" + echo "" > $dest/backup.xml + for disk in $(virsh -c qemu:///system domblklist $VM --details 2>/dev/null | awk '/disk/{print$3}'); do + volpath=$(virsh -c qemu:///system domblklist $VM --details | awk "/$disk/{print $4}" | sed 's/.*\///') + echo "" >> $dest/backup.xml + name="datadisk" + done + echo "" >> $dest/backup.xml + + # Start push backup + virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml > /dev/null 2>/dev/null + + # Backup domain information + virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null + virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null + virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null + virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null + + until virsh -c qemu:///system domjobinfo $VM --completed --keep-completed 2>/dev/null | grep "Completed" > /dev/null; do + sleep 5 + done + rm -f $dest/backup.xml + sync + + # Print statistics + virsh -c qemu:///system domjobinfo $VM --completed + du -sb $dest | cut -f1 + + umount $mount_point + rmdir $mount_point +} + +backup_stopped_vm() { + mount_operation + mkdir -p $dest + + IFS="," + + name="root" + for disk in $DISK_PATHS; do + volUuid="${disk##*/}" + qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 + name="datadisk" + done + sync + + ls -l --numeric-uid-gid $dest | awk '{print $5}' +} + +delete_backup() { + mount_operation + + rm -frv $dest + sync + umount $mount_point + rmdir $mount_point +} + +mount_operation() { + mount_point=$(mktemp -d -t csbackup.XXXXX) + dest="$mount_point/${BACKUP_DIR}" + mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) +} + +function usage { + echo "" + echo "Usage: $0 -o -v|--vm -t -s -m -p -d " + echo "" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--operation) + OP="$2" + shift + shift + ;; + -v|--vm) + VM="$2" + shift + shift + ;; + -t|--type) + NAS_TYPE="$2" + shift + shift + ;; + -s|--storage) + NAS_ADDRESS="$2" + shift + shift + ;; + -m|--mount) + MOUNT_OPTS="$2" + shift + shift + ;; + -p|--path) + BACKUP_DIR="$2" + shift + shift + ;; + -d|--diskpaths) + DISK_PATHS="$2" + shift + shift + ;; + -h|--help) + usage + shift + ;; + *) + echo "Invalid option: $1" + usage + ;; + esac +done + +if [ "$OP" = "backup" ]; then + STATE=$(virsh -c qemu:///system list | grep $VM | awk '{print $3}') + if [ "$STATE" = "running" ]; then + backup_running_vm + else + backup_stopped_vm + fi +elif [ "$OP" = "delete" ]; then + delete_backup +fi diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index c25d7b920c2d..e53de0c1d479 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -66,6 +66,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse; import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.BucketResponse; @@ -185,8 +186,10 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.BackupRepository; import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; import org.apache.cloudstack.config.ConfigurationSubGroup; @@ -489,6 +492,8 @@ public class ApiResponseHelper implements ResponseGenerator { UserDataDao userDataDao; @Inject VlanDetailsDao vlanDetailsDao; + @Inject + BackupRepositoryDao backupRepositoryDao; @Inject ObjectStoreDao _objectStoreDao; @@ -5284,6 +5289,26 @@ public BucketResponse createBucketResponse(Bucket bucket) { return bucketResponse; } + @Override + public BackupRepositoryResponse createBackupRepositoryResponse(BackupRepository backupRepository) { + BackupRepositoryResponse response = new BackupRepositoryResponse(); + response.setName(backupRepository.getName()); + response.setId(backupRepository.getUuid()); + response.setCreated(backupRepository.getCreated()); + response.setAddress(backupRepository.getAddress()); + response.setProviderName(backupRepository.getProvider()); + response.setType(backupRepository.getType()); + response.setMountOptions(backupRepository.getMountOptions()); + response.setCapacityBytes(backupRepository.getCapacityBytes()); + response.setObjectName("backuprepository"); + DataCenter zone = ApiDBUtils.findZoneById(backupRepository.getZoneId()); + if (zone != null) { + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + } + return response; + } + @Override public SharedFSResponse createSharedFSResponse(ResponseView view, SharedFS sharedFS) { SharedFSJoinVO sharedFSView = ApiDBUtils.newSharedFSView(sharedFS); diff --git a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java index ff588d064791..c27adc59fde0 100644 --- a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java @@ -355,15 +355,14 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, vm.setPowerState(VirtualMachine.PowerState.PowerOff); _instanceDao.update(vm.getId(), vm); } - for ( Backup.VolumeInfo VMVolToRestore : vm.getBackupVolumeList()) { + for (Backup.VolumeInfo VMVolToRestore : vm.getBackupVolumeList()) { VolumeVO volume = _volumeDao.findByUuidIncludingRemoved(VMVolToRestore.getUuid()); volume.setState(Volume.State.Ready); _volumeDao.update(volume.getId(), volume); if (VMVolToRestore.getType() == Volume.Type.ROOT) { _volumeDao.update(volume.getId(), volume); _volumeDao.attachVolume(volume.getId(), vm.getId(), 0L); - } - else if ( VMVolToRestore.getType() == Volume.Type.DATADISK) { + } else if (VMVolToRestore.getType() == Volume.Type.DATADISK) { List vmVolumes = _volumeDao.findByInstance(vm.getId()); _volumeDao.update(volume.getId(), volume); _volumeDao.attachVolume(volume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes)); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 8a6d13f83707..a392cecbeb4f 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -53,6 +53,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.direct.download.DirectDownloadHelper; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; @@ -348,6 +349,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic protected ProjectManager projectManager; @Inject protected StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + private BackupDao backupDao; protected Gson _gson; @@ -2616,7 +2619,8 @@ private void checkForDevicesInCopies(Long vmId, UserVmVO vm) { } // if target VM has backups - if (vm.getBackupOfferingId() != null || vm.getBackupVolumeList().size() > 0) { + List backups = backupDao.listByVmId(vm.getDataCenterId(), vm.getId()); + if (vm.getBackupOfferingId() != null && !backups.isEmpty()) { throw new InvalidParameterValueException(String.format("Unable to attach volume to VM %s/%s, please specify a VM that does not have any backups", vm.getName(), vm.getUuid())); } } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 47dd28f822dd..b86b65bd465d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -26,7 +26,9 @@ import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; +import java.util.stream.Collectors; +import com.amazonaws.util.CollectionUtils; import com.cloud.storage.VolumeApiService; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.UserVmManager; @@ -54,6 +56,9 @@ import org.apache.cloudstack.api.command.user.backup.RestoreBackupCmd; import org.apache.cloudstack.api.command.user.backup.RestoreVolumeFromBackupAndAttachToVMCmd; import org.apache.cloudstack.api.command.user.backup.UpdateBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupScheduleDao; @@ -238,7 +243,11 @@ public Pair, Integer> listBackupOfferings(final ListBackupO SearchBuilder sb = backupOfferingDao.createSearchBuilder(); sb.and("zone_id", sb.entity().getZoneId(), SearchCriteria.Op.EQ); sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); - + CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + if (Account.Type.NORMAL == caller.getType()) { + sb.and("user_backups_allowed", sb.entity().isUserDrivenBackupAllowed(), SearchCriteria.Op.EQ); + } final SearchCriteria sc = sb.create(); if (zoneId != null) { @@ -248,6 +257,9 @@ public Pair, Integer> listBackupOfferings(final ListBackupO if (keyword != null) { sc.setParameters("name", "%" + keyword + "%"); } + if (Account.Type.NORMAL == caller.getType()) { + sc.setParameters("user_backups_allowed", true); + } Pair, Integer> result = backupOfferingDao.searchAndCount(sc, searchFilter); return new Pair<>(new ArrayList<>(result.first()), result.second()); } @@ -267,7 +279,7 @@ public boolean deleteBackupOffering(final Long offeringId) { return backupOfferingDao.remove(offering.getId()); } - private String createVolumeInfoFromVolumes(List vmVolumes) { + public static String createVolumeInfoFromVolumes(List vmVolumes) { List list = new ArrayList<>(); for (VolumeVO vol : vmVolumes) { list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize())); @@ -342,6 +354,7 @@ public VMInstanceVO doInTransaction(final TransactionStatus status) { backupProvider.getName(), backupProvider.getClass().getSimpleName(), e.getMessage()); logger.error(msg); logger.debug(msg, e); + return null; } return vm; } @@ -448,7 +461,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { throw new InvalidParameterValueException("Invalid schedule: " + cmd.getSchedule() + " for interval type: " + cmd.getIntervalType()); } - final BackupScheduleVO schedule = backupScheduleDao.findByVM(vmId); + final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime)); } @@ -462,12 +475,12 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { } @Override - public BackupSchedule listBackupSchedule(final Long vmId) { + public List listBackupSchedule(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); - return backupScheduleDao.findByVM(vmId); + return backupScheduleDao.listByVM(vmId).stream().map(BackupSchedule.class::cast).collect(Collectors.toList()); } @Override @@ -632,12 +645,24 @@ public boolean restoreBackup(final Long backupId) { !vm.getState().equals(VirtualMachine.State.Destroyed)) { throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup"); } + // This is done to handle historic backups if any with Veeam / Networker plugins + List backupVolumes = CollectionUtils.isNullOrEmpty(backup.getBackedUpVolumes()) ? + vm.getBackupVolumeList() : backup.getBackedUpVolumes(); + List vmVolumes = volumeDao.findByInstance(vm.getId()); + if (vmVolumes.size() != backupVolumes.size()) { + throw new CloudRuntimeException("Unable to restore VM with the current backup as the backup has different number of disks as the VM"); + } - final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(vm.getBackupOfferingId()); + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(vm.getBackupOfferingId()); + String errorMessage = "Failed to find backup offering of the VM backup."; if (offering == null) { - throw new CloudRuntimeException("Failed to find backup offering of the VM backup."); + logger.warn(errorMessage); + } + logger.debug("Attempting to get backup offering from VM backup"); + offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (offering == null) { + throw new CloudRuntimeException(errorMessage); } - String backupDetailsInMessage = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(backup, "uuid", "externalId", "vmId", "type", "status", "date"); tryRestoreVM(backup, vm, offering, backupDetailsInMessage); updateVolumeState(vm, Volume.Event.RestoreSucceeded, Volume.State.Ready); @@ -780,26 +805,32 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, throw new CloudRuntimeException("VM reference for the provided VM backup not found"); } accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vmFromBackup); + final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (offering == null) { + throw new CloudRuntimeException("Failed to find VM backup offering"); + } + + BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + VolumeVO backedUpVolume = volumeDao.findByUuid(backedUpVolumeUuid); + Pair restoreInfo; + if (!"nas".equals(offering.getProvider())) { + restoreInfo = getRestoreVolumeHostAndDatastore(vm); + } else { + restoreInfo = getRestoreVolumeHostAndDatastoreForNas(vm, backedUpVolume); + } - Pair restoreInfo = getRestoreVolumeHostAndDatastore(vm); HostVO host = restoreInfo.first(); StoragePoolVO datastore = restoreInfo.second(); logger.debug("Asking provider to restore volume " + backedUpVolumeUuid + " from backup " + backupId + " (with external ID " + backup.getExternalId() + ") and attach it to VM: " + vm.getUuid()); - final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); - if (offering == null) { - throw new CloudRuntimeException("Failed to find VM backup offering"); - } - - BackupProvider backupProvider = getBackupProvider(offering.getProvider()); logger.debug(String.format("Trying to restore volume using host private IP address: [%s].", host.getPrivateIpAddress())); String[] hostPossibleValues = {host.getPrivateIpAddress(), host.getName()}; String[] datastoresPossibleValues = {datastore.getUuid(), datastore.getName()}; - Pair result = restoreBackedUpVolume(backedUpVolumeUuid, backup, backupProvider, hostPossibleValues, datastoresPossibleValues); + Pair result = restoreBackedUpVolume(backedUpVolumeUuid, backup, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); if (BooleanUtils.isFalse(result.first())) { throw new CloudRuntimeException(String.format("Error restoring volume [%s] of VM [%s] to host [%s] using backup provider [%s] due to: [%s].", @@ -813,7 +844,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, } protected Pair restoreBackedUpVolume(final String backedUpVolumeUuid, final BackupVO backup, BackupProvider backupProvider, String[] hostPossibleValues, - String[] datastoresPossibleValues) { + String[] datastoresPossibleValues, VMInstanceVO vm) { Pair result = new Pair<>(false, ""); for (String hostData : hostPossibleValues) { for (String datastoreData : datastoresPossibleValues) { @@ -821,7 +852,7 @@ protected Pair restoreBackedUpVolume(final String backedUpVolum backedUpVolumeUuid, hostData, datastoreData)); try { - result = backupProvider.restoreBackedUpVolume(backup, backedUpVolumeUuid, hostData, datastoreData); + result = backupProvider.restoreBackedUpVolume(backup, backedUpVolumeUuid, hostData, datastoreData, new Pair<>(vm.getName(), vm.getState())); if (BooleanUtils.isTrue(result.first())) { return result; @@ -874,6 +905,15 @@ private Pair getRestoreVolumeHostAndDatastore(VMInstanceV return new Pair<>(hostVO, storagePoolVO); } + private Pair getRestoreVolumeHostAndDatastoreForNas(VMInstanceVO vm, VolumeVO backedVolume) { + Long poolId = backedVolume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + HostVO hostVO = vm.getHostId() == null ? + getFirstHostFromStoragePool(storagePoolVO) : + hostDao.findById(vm.getHostId()); + return new Pair<>(hostVO, storagePoolVO); + } + /** * Find a host from storage pool access */ @@ -976,6 +1016,9 @@ public List> getCommands() { cmdList.add(RestoreBackupCmd.class); cmdList.add(DeleteBackupCmd.class); cmdList.add(RestoreVolumeFromBackupAndAttachToVMCmd.class); + cmdList.add(AddBackupRepositoryCmd.class); + cmdList.add(DeleteBackupRepositoryCmd.class); + cmdList.add(ListBackupRepositoriesCmd.class); return cmdList; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupRepositoryServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupRepositoryServiceImpl.java new file mode 100644 index 000000000000..5eb6538eaf51 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupRepositoryServiceImpl.java @@ -0,0 +1,114 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; +import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class BackupRepositoryServiceImpl extends ManagerBase implements BackupRepositoryService { + + @Inject + private BackupRepositoryDao repositoryDao; + @Inject + private BackupOfferingDao backupOfferingDao; + @Inject + private BackupDao backupDao; + @Inject + private AccountManager accountManager; + + @Override + public BackupRepository addBackupRepository(AddBackupRepositoryCmd cmd) { + BackupRepositoryVO repository = new BackupRepositoryVO(cmd.getZoneId(), cmd.getProvider(), cmd.getName(), + cmd.getType(), cmd.getAddress(), cmd.getMountOptions(), cmd.getCapacityBytes()); + return repositoryDao.persist(repository); + } + + @Override + public boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd) { + BackupRepositoryVO backupRepositoryVO = repositoryDao.findById(cmd.getId()); + if (Objects.isNull(backupRepositoryVO)) { + logger.debug("Backup repository appears to already be deleted"); + return false; + } + BackupOffering offeringVO = backupOfferingDao.findByExternalId(backupRepositoryVO.getUuid(), backupRepositoryVO.getZoneId()); + if (Objects.nonNull(offeringVO)) { + List backups = backupDao.listByOfferingId(offeringVO.getId()); + if (!backups.isEmpty()) { + throw new CloudRuntimeException("Failed to delete backup repository as there are backups present on it"); + } + } + return repositoryDao.remove(backupRepositoryVO.getId()); + } + + @Override + public Pair, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd) { + Long zoneId = accountManager.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), cmd.getZoneId()); + Long id = cmd.getId(); + String name = cmd.getName(); + String provider = cmd.getProvider(); + String keyword = cmd.getKeyword(); + + SearchBuilder sb = repositoryDao.createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.and("provider", sb.entity().getProvider(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + if (keyword != null) { + SearchCriteria ssc = repositoryDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("provider", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + if (Objects.nonNull(id)) { + sc.setParameters("id", id); + } + if (Objects.nonNull(name)) { + sc.setParameters("name", name); + } + if (Objects.nonNull(zoneId)) { + sc.setParameters("zoneId", zoneId); + } + if (Objects.nonNull(provider)) { + sc.setParameters("provider", provider); + } + + // search Store details by ids + Pair, Integer> repositoryVOPair = repositoryDao.searchAndCount(sc, null); + return new Pair<>(new ArrayList<>(repositoryVOPair.first()), repositoryVOPair.second()); + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 4a809b5ded3b..3237d11d1c7a 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -338,6 +338,8 @@ + + diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index da48cc2666a6..42bfb416fe9e 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -145,6 +146,8 @@ public class VolumeApiServiceImplTest { @Mock private VolumeDao volumeDaoMock; @Mock + private BackupDao backupDaoMock; + @Mock private AccountManager accountManagerMock; @Mock private UserVmDao userVmDaoMock; @@ -632,7 +635,7 @@ public void testResourceLimitCheckForUploadedVolume() throws NoSuchFieldExceptio when(vm.getState()).thenReturn(State.Running); when(vm.getDataCenterId()).thenReturn(34L); when(vm.getBackupOfferingId()).thenReturn(null); - when(vm.getBackupVolumeList()).thenReturn(Collections.emptyList()); + when(backupDaoMock.listByVmId(anyLong(), anyLong())).thenReturn(Collections.emptyList()); when(volumeDaoMock.findByInstanceAndType(anyLong(), any(Volume.Type.class))).thenReturn(new ArrayList<>(10)); when(volumeDataFactoryMock.getVolume(9L)).thenReturn(volumeToAttach); when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 7726af09361e..3bf1fb97e4d0 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -47,6 +47,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -158,65 +159,88 @@ public void testUpdateBackupOfferingSuccess() { @Test public void restoreBackedUpVolumeTestHostIpAndDatastoreUuid() { BackupVO backupVO = new BackupVO(); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; + String vmName = "i-2-3-VM"; + VirtualMachine.State vmState = VirtualMachine.State.Running; + Mockito.when(vm.getName()).thenReturn(vmName); + Mockito.when(vm.getState()).thenReturn(vmState); + Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"))).thenReturn(new Pair(Boolean.TRUE, "Success")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues); + Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success", restoreBackedUpVolume.second()); Mockito.verify(backupProvider, times(1)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyString(), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostIpAndDatastoreName() { BackupVO backupVO = new BackupVO(); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; - + String vmName = "i-2-3-VM"; + VirtualMachine.State vmState = VirtualMachine.State.Running; + Mockito.when(vm.getName()).thenReturn(vmName); + Mockito.when(vm.getState()).thenReturn(vmState); + Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"))).thenReturn(new Pair(Boolean.TRUE, "Success2")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues); + Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success2")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success2", restoreBackedUpVolume.second()); Mockito.verify(backupProvider, times(2)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyString(), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostNameAndDatastoreUuid() { BackupVO backupVO = new BackupVO(); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; + String vmName = "i-2-3-VM"; + VirtualMachine.State vmState = VirtualMachine.State.Running; + Mockito.when(vm.getName()).thenReturn(vmName); + Mockito.when(vm.getState()).thenReturn(vmState); + Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"))).thenReturn(new Pair(Boolean.TRUE, "Success3")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues); + Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState) )).thenReturn(new Pair(Boolean.TRUE, "Success3")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success3", restoreBackedUpVolume.second()); Mockito.verify(backupProvider, times(3)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyString(), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostAndDatastoreName() { BackupVO backupVO = new BackupVO(); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; + String vmName = "i-2-3-VM"; + VirtualMachine.State vmState = VirtualMachine.State.Running; + Mockito.when(vm.getName()).thenReturn(vmName); + Mockito.when(vm.getState()).thenReturn(vmState); + Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("datastore-name"))).thenReturn(new Pair(Boolean.TRUE, "Success4")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues); + Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success4", restoreBackedUpVolume.second()); Mockito.verify(backupProvider, times(4)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyString(), any(Pair.class)); } @Test diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 529bba39c2fe..83e5561dc4d5 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -409,9 +409,12 @@ "label.backup.offering.assign": "Assign Instance to backup offering", "label.backup.offering.remove": "Remove Instance from backup offering", "label.backup.offerings": "Backup offerings", +"label.backup.repository": "Backup Repository", "label.backup.restore": "Restore Instance backup", "label.backupofferingid": "Backup offering", "label.backupofferingname": "Backup offering", +"label.backup.repository.add": "Add backup repository", +"label.backup.repository.remove": "Remove backup repository", "label.balance": "Balance", "label.bandwidth": "Bandwidth", "label.baremetal.dhcp.devices": "Bare metal DHCP devices", @@ -2558,6 +2561,7 @@ "message.action.create.snapshot.from.vmsnapshot": "Please confirm that you want to create Snapshot from Instance Snapshot", "message.action.delete.autoscale.vmgroup": "Please confirm that you want to delete this autoscaling group.", "message.action.delete.backup.offering": "Please confirm that you want to delete this backup offering?", +"message.action.delete.backup.repository": "Please confirm that you want to delete this backup repository?", "message.action.delete.cluster": "Please confirm that you want to delete this cluster.", "message.action.delete.domain": "Please confirm that you want to delete this domain.", "message.action.delete.external.firewall": "Please confirm that you would like to remove this external firewall. Warning: If you are planning to add back the same external firewall, you must reset usage data on the device.", diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index a7e805b54439..a95927f00cf0 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -50,6 +50,10 @@ {{ $toLocaleDate(text) }} + + diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 6d694f4f6d66..e8b0db974e6b 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -181,7 +181,7 @@ -