Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NAS B&R Plugin enhancements #9666

Open
wants to merge 25 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
64db486
NAS B&R Plugin enhancements
Pearl1594 Sep 10, 2024
d5a259d
Prevent printing mount opts which may include password by removing fr…
Pearl1594 Sep 20, 2024
e2cb774
Merge branch 'main' of https://github.com/apache/cloudstack into nas-…
Pearl1594 Sep 20, 2024
6dfb0ea
revert marvin change
Pearl1594 Sep 20, 2024
b1ccf9d
add sanity checks to validate minimum qemu and libvirt versions
Pearl1594 Sep 23, 2024
343e7dc
check is user running script is part of libvirt group
Pearl1594 Sep 23, 2024
f2f81c8
revert changes of retore expunged VM
Pearl1594 Sep 23, 2024
be9eba3
add code coverage ignore file
Pearl1594 Sep 30, 2024
585126b
remove check
Pearl1594 Sep 30, 2024
d024a96
issue with listing schedules and add defensive checks
Pearl1594 Sep 30, 2024
5c23f4b
redirect logs to agent log file
Pearl1594 Oct 1, 2024
0760ef5
add some more debugging
Pearl1594 Oct 1, 2024
bf19dea
remove test file
Pearl1594 Oct 1, 2024
5430080
Merge branch 'main' of https://github.com/apache/cloudstack into nas-…
Pearl1594 Oct 1, 2024
5bfed06
prevent deletion of cks cluster when vms associated to a backup offering
Pearl1594 Oct 1, 2024
c087de4
Merge branch '4.19'
DaanHoogland Oct 2, 2024
3ae3601
delete all snapshot policies when bkp offering is disassociated from …
Pearl1594 Oct 4, 2024
d6181d5
Fix `updateTemplatePermission` when the UI is set to a language other…
lucas-a-martins Oct 4, 2024
59464be
Add nobrl in the mountopts for cifs file system
Pearl1594 Oct 8, 2024
862211c
Fix restoration of VM / volumes with cifs
Pearl1594 Oct 8, 2024
7caa908
add cifs utils for el8
Pearl1594 Oct 8, 2024
482c2c9
add cifs-utils for ubuntu cloudstack-agent
Pearl1594 Oct 8, 2024
75e5c0c
Merge branch 'main' of https://github.com/apache/cloudstack into nas-…
Pearl1594 Oct 8, 2024
f6f0ba6
syntax error
Pearl1594 Oct 9, 2024
9a6e0e3
Merge branch 'main' into nas-br-partb
DaanHoogland Oct 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE
List<BackupSchedule> schedules = backupManager.listBackupSchedule(getVmId());
ListResponse<BackupScheduleResponse> response = new ListResponse<>();
List<BackupScheduleResponse> scheduleResponses = new ArrayList<>();
if (CollectionUtils.isNullOrEmpty(schedules)) {
if (!CollectionUtils.isNullOrEmpty(schedules)) {
for (BackupSchedule schedule : schedules) {
scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,6 @@ public class BackupRepositoryResponse extends BaseResponse {
@Param(description = "backup type")
private String type;

@SerializedName(ApiConstants.MOUNT_OPTIONS)
@Param(description = "mount options for the backup repository")
private String mountOptions;

@SerializedName(ApiConstants.CAPACITY_BYTES)
@Param(description = "capacity of the backup repository")
private Long capacityBytes;
Expand Down Expand Up @@ -112,14 +108,6 @@ public void setAddress(String address) {
this.address = address;
}

public String getMountOptions() {
return mountOptions;
}

public void setMountOptions(String mountOptions) {
this.mountOptions = mountOptions;
}

public String getProviderName() {
return providerName;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -373,8 +373,12 @@
Long vmBackupSize = 0L;
Long vmBackupProtectedSize = 0L;
for (final Backup backup: backupDao.listByVmId(null, vm.getId())) {
vmBackupSize += backup.getSize();
vmBackupProtectedSize += backup.getProtectedSize();
if (Objects.nonNull(backup.getSize())) {
vmBackupSize += backup.getSize();

Check warning on line 377 in plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

View check run for this annotation

Codecov / codecov/patch

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java#L377

Added line #L377 was not covered by tests
}
if (Objects.nonNull(backup.getProtectedSize())) {
vmBackupProtectedSize += backup.getProtectedSize();

Check warning on line 380 in plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

View check run for this annotation

Codecov / codecov/patch

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java#L380

Added line #L380 was not covered by tests
}
}
Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize);
LOG.debug(String.format("Metrics for VM [uuid: %s, name: %s] is [backup size: %s, data size: %s].", vm.getUuid(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
Expand Down Expand Up @@ -1468,6 +1469,10 @@
}

List<KubernetesClusterVmMapVO> vmMapList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId);
List<VMInstanceVO> vms = vmMapList.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList());

Check warning on line 1472 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java#L1472

Added line #L1472 was not covered by tests
if (checkIfVmsAssociatedWithBackupOffering(vms)) {
throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering");

Check warning on line 1474 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java#L1474

Added line #L1474 was not covered by tests
}
Comment on lines +1473 to +1475
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not throw the exception from the new method?

for (KubernetesClusterVmMapVO vmMap : vmMapList) {
try {
userVmService.destroyVm(vmMap.getVmId(), expunge);
Expand All @@ -1490,6 +1495,15 @@
}
}

public static boolean checkIfVmsAssociatedWithBackupOffering(List<VMInstanceVO> vms) {

Check warning on line 1498 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java#L1498

Added line #L1498 was not covered by tests
for(VMInstanceVO vm : vms) {
if (Objects.nonNull(vm.getBackupOfferingId())) {
return true;

Check warning on line 1501 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java#L1501

Added line #L1501 was not covered by tests
}
}
return false;
}

Check warning on line 1505 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java#L1503-L1505

Added lines #L1503 - L1505 were not covered by tests

@Override
public ListResponse<KubernetesClusterResponse> listKubernetesClusters(ListKubernetesClustersCmd cmd) {
if (!KubernetesServiceEnabled.value()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,10 @@
init();
validateClusterSate();
this.clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
List<VMInstanceVO> vms = this.clusterVMs.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList());

Check warning on line 248 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java#L248

Added line #L248 was not covered by tests
if (KubernetesClusterManagerImpl.checkIfVmsAssociatedWithBackupOffering(vms)) {
throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering");

Check warning on line 250 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java

View check run for this annotation

Codecov / codecov/patch

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java#L250

Added line #L250 was not covered by tests
}
boolean cleanupNetwork = true;
final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "networkCleanup");
if (clusterDetails != null) {
Expand Down
65 changes: 63 additions & 2 deletions scripts/vm/hypervisor/kvm/nasbackup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,58 @@ NAS_ADDRESS=""
MOUNT_OPTS=""
BACKUP_DIR=""
DISK_PATHS=""
logFile="/var/log/cloudstack/agent/agent.log"

log() {
[[ "$verb" -eq 1 ]] && builtin echo "$@"
if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then
builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile"
else
builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile"
fi
}

vercomp() {
local IFS=.
local i ver1=($1) ver2=($3)

# Compare each segment of the version numbers
for ((i=0; i<${#ver1[@]}; i++)); do
if [[ -z ${ver2[i]} ]]; then
ver2[i]=0
fi

if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 0 # Version 1 is greater
elif ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2 # Version 2 is greater
fi
done
return 0 # Versions are equal
}

sanity_checks() {
hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}')
libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1)
apiVersion=$(virsh version | grep API | awk '{print $(NF)}')

# Compare qemu version (hvVersion >= 4.2.0)
vercomp "$hvVersion" ">=" "4.2.0"
hvStatus=$?

# Compare libvirt version (libvVersion >= 7.2.0)
vercomp "$libvVersion" ">=" "7.2.0"
libvStatus=$?

if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then
log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]"
else
echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0"
exit 1
fi

log -ne "Environment Sanity Checks successfully passed"
}

### Operation methods ###

Expand Down Expand Up @@ -79,7 +131,7 @@ backup_stopped_vm() {
name="root"
for disk in $DISK_PATHS; do
volUuid="${disk##*/}"
qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2
qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 | tee -a "$logFile"
name="datadisk"
done
sync
Expand All @@ -99,7 +151,13 @@ delete_backup() {
mount_operation() {
mount_point=$(mktemp -d -t csbackup.XXXXX)
dest="$mount_point/${BACKUP_DIR}"
mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS})
mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) | tee -a "$logFile"
if [ $? -eq 0 ]; then
log -ne "Successfully mounted ${NAS_TYPE} store"
else
echo "Failed to mount ${NAS_TYPE} store"
exit 1
fi
}

function usage {
Expand Down Expand Up @@ -157,6 +215,9 @@ while [[ $# -gt 0 ]]; do
esac
done

# Perform Initial sanity checks
sanity_checks

if [ "$OP" = "backup" ]; then
STATE=$(virsh -c qemu:///system list | grep $VM | awk '{print $3}')
if [ "$STATE" = "running" ]; then
Expand Down
1 change: 0 additions & 1 deletion server/src/main/java/com/cloud/api/ApiResponseHelper.java
Original file line number Diff line number Diff line change
Expand Up @@ -5439,7 +5439,6 @@ public BackupRepositoryResponse createBackupRepositoryResponse(BackupRepository
response.setAddress(backupRepository.getAddress());
response.setProviderName(backupRepository.getProvider());
response.setType(backupRepository.getType());
response.setMountOptions(backupRepository.getMountOptions());
response.setCapacityBytes(backupRepository.getCapacityBytes());
response.setObjectName("backuprepository");
DataCenter zone = ApiDBUtils.findZoneById(backupRepository.getZoneId());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
import org.apache.cloudstack.backup.dao.BackupScheduleDao;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher;
import org.apache.cloudstack.framework.jobs.AsyncJobManager;
Expand Down Expand Up @@ -162,8 +161,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
private VirtualMachineManager virtualMachineManager;
@Inject
private VolumeApiService volumeApiService;
@Inject
private VolumeOrchestrationService volumeOrchestrationService;

private AsyncJobDispatcher asyncJobDispatcher;
private Timer backupTimer;
Expand Down Expand Up @@ -623,6 +620,7 @@ public boolean restoreBackup(final Long backupId) {
!vm.getState().equals(VirtualMachine.State.Destroyed)) {
throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup");
}

// This is done to handle historic backups if any with Veeam / Networker plugins
List<Backup.VolumeInfo> backupVolumes = CollectionUtils.isNullOrEmpty(backup.getBackedUpVolumes()) ?
vm.getBackupVolumeList() : backup.getBackedUpVolumes();
Expand Down
2 changes: 1 addition & 1 deletion ui/src/config/section/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ export default {
],
mapping: {
type: {
options: ['nfs']
options: ['nfs', 'cifs']
},
provider: {
value: (record) => { return 'nas' }
Expand Down
Loading