Skip to content

Commit

Permalink
Merge remote-tracking branch 'apache/4.18'
Browse files Browse the repository at this point in the history
  • Loading branch information
weizhouapache committed Jun 27, 2023
2 parents c3718ab + 83dca2b commit 41403c9
Show file tree
Hide file tree
Showing 21 changed files with 1,266 additions and 30 deletions.
2 changes: 1 addition & 1 deletion api/src/main/java/com/cloud/storage/Storage.java
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ public static enum StoragePoolType {
ManagedNFS(true, false, false),
Linstor(true, true, false),
DatastoreCluster(true, true, false), // for VMware, to abstract pool of clusters
StorPool(true, true, false);
StorPool(true, true, true);

private final boolean shared;
private final boolean overprovisioning;
Expand Down
2 changes: 2 additions & 0 deletions engine/schema/src/main/java/com/cloud/host/dao/HostDao.java
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat

List<HostVO> findByClusterId(Long clusterId);

List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId);

/**
* Returns hosts that are 'Up' and 'Enabled' from the given Data Center/Zone
*/
Expand Down
26 changes: 26 additions & 0 deletions engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -1158,6 +1158,32 @@ public List<HostVO> findByClusterId(Long clusterId) {
return listBy(sc);
}

@Override
public List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId) {
SearchBuilder<DetailVO> hostCapabilitySearch = _detailsDao.createSearchBuilder();
DetailVO tagEntity = hostCapabilitySearch.entity();
hostCapabilitySearch.and("capability", tagEntity.getName(), SearchCriteria.Op.EQ);
hostCapabilitySearch.and("value", tagEntity.getValue(), SearchCriteria.Op.EQ);

SearchBuilder<HostVO> hostSearch = createSearchBuilder();
HostVO entity = hostSearch.entity();
hostSearch.and("cluster", entity.getClusterId(), SearchCriteria.Op.EQ);
hostSearch.and("status", entity.getStatus(), SearchCriteria.Op.EQ);
hostSearch.join("hostCapabilitySearch", hostCapabilitySearch, entity.getId(), tagEntity.getHostId(), JoinBuilder.JoinType.INNER);

SearchCriteria<HostVO> sc = hostSearch.create();
sc.setJoinParameters("hostCapabilitySearch", "value", Boolean.toString(true));
sc.setJoinParameters("hostCapabilitySearch", "capability", Host.HOST_VOLUME_ENCRYPTION);

if (clusterId != null) {
sc.setParameters("cluster", clusterId);
}
sc.setParameters("status", Status.Up.toString());
sc.setParameters("resourceState", ResourceState.Enabled.toString());

return listBy(sc);
}

@Override
public HostVO findByPublicIp(String publicIp) {
SearchCriteria<HostVO> sc = PublicIpAddressSearch.create();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,4 +147,6 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
*/
List<VolumeVO> findByDiskOfferingId(long diskOfferingId);
VolumeVO getInstanceRootVolume(long instanceId);

void updateAndRemoveVolume(VolumeVO volume);
}
Original file line number Diff line number Diff line change
Expand Up @@ -766,4 +766,15 @@ public VolumeVO getInstanceRootVolume(long instanceId) {
sc.setParameters("vType", Volume.Type.ROOT);
return findOneBy(sc);
}

@Override
public void updateAndRemoveVolume(VolumeVO volume) {
if (volume.getState() != Volume.State.Destroy) {
volume.setState(Volume.State.Destroy);
volume.setPoolId(null);
volume.setInstanceId(null);
update(volume.getId(), volume);
remove(volume.getId());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
-- Schema upgrade from 4.18.0.0 to 4.18.1.0
--;

-- Update conserve_mode of the default network offering for Tungsten Fabric (this fixes issue #7241)
UPDATE `cloud`.`network_offerings` SET conserve_mode = 0 WHERE unique_name ='DefaultTungstenFarbicNetworkOffering';

-- Add Windows Server 2022 guest OS and mappings
CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'KVM', 'default', 'Windows Server 2022 (64-bit)');
CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '7.0', 'windows2019srvNext_64Guest');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
networkOfferingVO = new NetworkOfferingVO(NETWORKOFFERING,
"Default offering for Tungsten-Fabric Network", Networks.TrafficType.Guest, false, false,
null, null, true, NetworkOffering.Availability.Optional, null, Network.GuestType.Isolated,
true, false, false, false, true, false);
false, false, false, false, true, false);
networkOfferingVO.setForTungsten(true);
networkOfferingVO.setState(NetworkOffering.State.Enabled);
networkOfferingDao.persist(networkOfferingVO);
Expand Down
9 changes: 9 additions & 0 deletions plugins/storage/volume/storpool/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -342,3 +342,12 @@ Max IOPS are kept in StorPool's volumes with the help of custom service offering
corresponding system disk offering.

CloudStack has no way to specify max BW. Do they want to be able to specify max BW only is sufficient.

## Supported operations for Volume encryption

Supported Virtual machine operations - live migration of VM to another host, virtual machine snapshots (group snapshot without memory), revert VM snapshot, delete VM snapshot

Supported Volume operations - attach/detach volume, live migrate volume between two StorPool primary storages, volume snapshot, delete snapshot, revert snapshot

Note: volume snapshot are allowed only when `sp.bypass.secondary.storage` is set to `true`. This means that the snapshots are not backed up to secondary storage

Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.agent.api.storage;

import org.apache.cloudstack.storage.to.VolumeObjectTO;

import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;

public class StorPoolSetVolumeEncryptionAnswer extends Answer {
private VolumeObjectTO volume;

public StorPoolSetVolumeEncryptionAnswer(Command command, boolean success, String details) {
super(command, success, details);
}

public StorPoolSetVolumeEncryptionAnswer(VolumeObjectTO volume) {
super();
this.volume = volume;
this.result = true;
}

public VolumeObjectTO getVolume() {
return volume;
}

public void setVolume(VolumeObjectTO volume) {
this.volume = volume;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.agent.api.storage;

import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.cloudstack.storage.to.VolumeObjectTO;

public class StorPoolSetVolumeEncryptionCommand extends StorageSubSystemCommand {
private boolean isDataDisk;
private VolumeObjectTO volumeObjectTO;
private String srcVolumeName;

public StorPoolSetVolumeEncryptionCommand(VolumeObjectTO volumeObjectTO, String srcVolumeName,
boolean isDataDisk) {
this.volumeObjectTO = volumeObjectTO;
this.srcVolumeName = srcVolumeName;
this.isDataDisk = isDataDisk;
}

public VolumeObjectTO getVolumeObjectTO() {
return volumeObjectTO;
}

public void setVolumeObjectTO(VolumeObjectTO volumeObjectTO) {
this.volumeObjectTO = volumeObjectTO;
}

public void setIsDataDisk(boolean isDataDisk) {
this.isDataDisk = isDataDisk;
}

public boolean isDataDisk() {
return isDataDisk;
}

public String getSrcVolumeName() {
return srcVolumeName;
}

public void setSrcVolumeName(String srcVolumeName) {
this.srcVolumeName = srcVolumeName;
}

@Override
public void setExecuteInSequence(boolean inSeq) {
inSeq = false;
}

@Override
public boolean executeInSequence() {
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.hypervisor.kvm.resource.wrapper;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.cryptsetup.CryptSetup;
import org.apache.cloudstack.utils.cryptsetup.CryptSetupException;
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.libvirt.LibvirtException;

import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.StorPoolSetVolumeEncryptionAnswer;
import com.cloud.agent.api.storage.StorPoolSetVolumeEncryptionCommand;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.exception.CloudRuntimeException;

@ResourceWrapper(handles = StorPoolSetVolumeEncryptionCommand.class)
public class StorPoolSetVolumeEncryptionCommandWrapper extends
CommandWrapper<StorPoolSetVolumeEncryptionCommand, StorPoolSetVolumeEncryptionAnswer, LibvirtComputingResource> {
private static final Logger logger = Logger.getLogger(StorPoolSetVolumeEncryptionCommandWrapper.class);

@Override
public StorPoolSetVolumeEncryptionAnswer execute(StorPoolSetVolumeEncryptionCommand command,
LibvirtComputingResource serverResource) {
VolumeObjectTO volume = command.getVolumeObjectTO();
String srcVolumeName = command.getSrcVolumeName();
try {
StorPoolStorageAdaptor.attachOrDetachVolume("attach", "volume", volume.getPath());
KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr();
PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) volume.getDataStore();
KVMStoragePool pool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());
KVMPhysicalDisk disk = pool.getPhysicalDisk(volume.getPath());
if (command.isDataDisk()) {
encryptDataDisk(volume, disk);
} else {
disk = encryptRootDisk(command, volume, srcVolumeName, pool, disk);
}
logger.debug(String.format("StorPoolSetVolumeEncryptionCommandWrapper disk=%s", disk));
} catch (Exception e) {
new Answer(command, e);
} finally {
StorPoolStorageAdaptor.attachOrDetachVolume("detach", "volume", volume.getPath());
volume.clearPassphrase();
}
return new StorPoolSetVolumeEncryptionAnswer(volume);
}

private KVMPhysicalDisk encryptRootDisk(StorPoolSetVolumeEncryptionCommand command, VolumeObjectTO volume,
String srcVolumeName, KVMStoragePool pool, KVMPhysicalDisk disk) {
StorPoolStorageAdaptor.attachOrDetachVolume("attach", "snapshot", srcVolumeName);
KVMPhysicalDisk srcVolume = pool.getPhysicalDisk(srcVolumeName);
disk = copyPhysicalDisk(srcVolume, disk, command.getWait() * 1000, null, volume.getPassphrase());
disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS);
disk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
volume.setEncryptFormat(disk.getQemuEncryptFormat().toString());
StorPoolStorageAdaptor.attachOrDetachVolume("detach", "snapshot", srcVolumeName);
return disk;
}

private void encryptDataDisk(VolumeObjectTO volume, KVMPhysicalDisk disk) throws CryptSetupException {
CryptSetup crypt = new CryptSetup();
crypt.luksFormat(volume.getPassphrase(), CryptSetup.LuksType.LUKS, disk.getPath());
disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS);
disk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
volume.setEncryptFormat(disk.getQemuEncryptFormat().toString());
}

private KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, KVMPhysicalDisk destDisk, int timeout,
byte[] srcPassphrase, byte[] dstPassphrase) {

logger.debug("Copy physical disk with size: " + disk.getSize() + ", virtualsize: " + disk.getVirtualSize()
+ ", format: " + disk.getFormat());

destDisk.setVirtualSize(disk.getVirtualSize());
destDisk.setSize(disk.getSize());

QemuImg qemu = null;
QemuImgFile srcQemuFile = null;
QemuImgFile destQemuFile = null;
String srcKeyName = "sec0";
String destKeyName = "sec1";
List<QemuObject> qemuObjects = new ArrayList<>();
Map<String, String> options = new HashMap<>();

try (KeyFile srcKey = new KeyFile(srcPassphrase); KeyFile dstKey = new KeyFile(dstPassphrase)) {
qemu = new QemuImg(timeout, true, false);
String srcPath = disk.getPath();
String destPath = destDisk.getPath();

QemuImageOptions qemuImageOpts = new QemuImageOptions(srcPath);

srcQemuFile = new QemuImgFile(srcPath, disk.getFormat());
destQemuFile = new QemuImgFile(destPath);

if (srcKey.isSet()) {
qemuObjects.add(QemuObject.prepareSecretForQemuImg(disk.getFormat(), disk.getQemuEncryptFormat(),
srcKey.toString(), srcKeyName, options));
qemuImageOpts = new QemuImageOptions(disk.getFormat(), srcPath, srcKeyName);
}

if (dstKey.isSet()) {
qemu.setSkipZero(false);
destDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW);
destQemuFile.setFormat(QemuImg.PhysicalDiskFormat.LUKS);
qemuObjects.add(QemuObject.prepareSecretForQemuImg(destDisk.getFormat(), QemuObject.EncryptFormat.LUKS,
dstKey.toString(), destKeyName, options));
destDisk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS);
}

qemu.convert(srcQemuFile, destQemuFile, options, qemuObjects, qemuImageOpts, null, true);
logger.debug("Successfully converted source disk image " + srcQemuFile.getFileName()
+ " to StorPool volume: " + destDisk.getPath());

} catch (QemuImgException | LibvirtException | IOException e) {

String errMsg = String.format("Unable to convert/copy from %s to %s, due to: %s", disk.getName(),
destDisk.getName(), ((StringUtils.isEmpty(e.getMessage())) ? "an unknown error" : e.getMessage()));
logger.error(errMsg);
throw new CloudRuntimeException(errMsg, e);
}

return destDisk;
}
}
Loading

0 comments on commit 41403c9

Please sign in to comment.