Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions tests/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,23 @@ You can check the permissions for the corresponding service account using `kubec
kubectl auth can-i --as=virt-e2e-sa ...
```

### Default StorageClass
### Default Storage Class

Default storage class should be set in the cluster. Annotate a StorageClass with
storageclass.kubernetes.io/is-default-class to mark it as the default:
Default storage class should be set in the cluster. You can set a default storage class in the `global` module config:

```bash
$ kubectl get moduleconfigs.deckhouse.io global --output yaml | yq .spec
```
```yaml
settings:
defaultClusterStorageClass: linstor-thin-r1
```

$ kubectl annotate storageclass linstor-thin-r1 storageclass.kubernetes.io/is-default-class=true
### Immediate Storage Class
Some test cases depend on an immediate storage class. You can skip the immediate storage class check if a test case does not require it.

$ kubectl get storageclass linstor-thin-r1 -o yaml | less
...
metadata:
annotations:
storageclass.kubernetes.io/is-default-class: "true"
...
```bash
FOCUS="VirtualMachineVersions" SKIP_IMMEDIATE_SC_CHECK="yes" task e2e:run
```

### E2E configuration
Expand Down
3 changes: 2 additions & 1 deletion tests/e2e/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ type TestData struct {
}

type StorageClass struct {
VolumeBindingMode storagev1.VolumeBindingMode
DefaultStorageClass *storagev1.StorageClass
ImmediateStorageClass *storagev1.StorageClass
}

type ClusterTransport struct {
Expand Down
42 changes: 42 additions & 0 deletions tests/e2e/config/storageclass.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
Copyright 2025 Flant JSC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
"fmt"
"log"
"os"
)

// An immediate storage class is required for test cases where a `VirtualDisk` might not be attached to a `VirtualMachine` but should be in the Ready phase to operate with it.
// If a test case does not require the immediate storage class, this check can be skipped.
const SkipImmediateStorageClassCheckEnv = "SKIP_IMMEDIATE_SC_CHECK"

func CheckStorageClassOption() error {
env := os.Getenv(SkipImmediateStorageClassCheckEnv)
switch env {
case "yes", "":
return nil
default:
log.Printf("To skip the immediate storage class check, set %s=yes. If you don't intend to skip this check, leave the variable unset.\n", SkipImmediateStorageClassCheckEnv)
return fmt.Errorf("invalid value for the %s env: %q", SkipImmediateStorageClassCheckEnv, env)
}
}

func SkipImmediateStorageClassCheck() bool {
return os.Getenv(SkipImmediateStorageClassCheckEnv) == "yes"
}
1 change: 0 additions & 1 deletion tests/e2e/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ go 1.23.0
toolchain go1.23.7

require (
github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db
github.com/deckhouse/virtualization/api v0.0.0-20240923080356-bb5809dba578
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
github.com/onsi/ginkgo/v2 v2.21.0
Expand Down
2 changes: 0 additions & 2 deletions tests/e2e/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db h1:ArXEN2TeIX+cX4DUGUtHpRhkJpSzzk80PNqY4PGxNi0=
github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db/go.mod h1:6yz0RtbkLVJtK2DeuvgfaqBZRl5V5ax1WsfPF5pbnvo=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
Expand Down
97 changes: 31 additions & 66 deletions tests/e2e/images_creation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,51 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
storagev1 "k8s.io/api/storage/v1"

sdsrepvolv1 "github.com/deckhouse/sds-replicated-volume/api/v1alpha1"
virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/tests/e2e/config"
"github.com/deckhouse/virtualization/tests/e2e/ginkgoutil"
. "github.com/deckhouse/virtualization/tests/e2e/helper"
kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
)

var _ = Describe("Virtual images creation", ginkgoutil.CommonE2ETestDecorators(), func() {
var _ = Describe("VirtualImagesCreation", ginkgoutil.CommonE2ETestDecorators(), func() {
var (
immediateStorageClassName string // require for unattached virtual disk snapshots
testCaseLabel = map[string]string{"testcase": "images-creation"}
testCaseLabel = map[string]string{"testcase": "images-creation"}
)

BeforeEach(func() {
BeforeAll(func() {
if config.IsReusable() {
Skip("Test not available in REUSABLE mode: not supported yet.")
}

kustomization := fmt.Sprintf("%s/%s", conf.TestData.ImagesCreation, "kustomization.yaml")
ns, err := kustomize.GetNamespace(kustomization)
Expect(err).NotTo(HaveOccurred(), "%w", err)
conf.SetNamespace(ns)

Expect(conf.StorageClass.ImmediateStorageClass).NotTo(BeNil(), "immediate storage class cannot be nil; please set up the immediate storage class in the cluster")

defaultVolumeSnapshotClassName, err := GetVolumeSnapshotClassName(conf.StorageClass.DefaultStorageClass)
Expect(err).NotTo(HaveOccurred(), "cannot define default `VolumeSnapshotClass`\nstderr: %s", err)

virtualDisk := virtv2.VirtualDisk{}
vdFilePath := fmt.Sprintf("%s/vd/vd-alpine-http.yaml", conf.TestData.ImagesCreation)
err = UnmarshalResource(vdFilePath, &virtualDisk)
Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdFilePath, err)

virtualDisk.Spec.PersistentVolumeClaim.StorageClass = &conf.StorageClass.ImmediateStorageClass.Name
err = WriteYamlObject(vdFilePath, &virtualDisk)
Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdFilePath, err)

virtualDiskSnapshot := virtv2.VirtualDiskSnapshot{}
vdSnapshotFilePath := fmt.Sprintf("%s/vdsnapshot/vdsnapshot.yaml", conf.TestData.ImagesCreation)
err = UnmarshalResource(vdSnapshotFilePath, &virtualDiskSnapshot)
Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdSnapshotFilePath, err)

virtualDiskSnapshot.Spec.VolumeSnapshotClassName = defaultVolumeSnapshotClassName
err = WriteYamlObject(vdSnapshotFilePath, &virtualDiskSnapshot)
Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdSnapshotFilePath, err)
})

AfterEach(func() {
Expand All @@ -49,56 +74,6 @@ var _ = Describe("Virtual images creation", ginkgoutil.CommonE2ETestDecorators()
}
})

Context("Preparing the environment", func() {
It("sets the namespace", func() {
kustomization := fmt.Sprintf("%s/%s", conf.TestData.ImagesCreation, "kustomization.yaml")
ns, err := kustomize.GetNamespace(kustomization)
Expect(err).NotTo(HaveOccurred(), "%w", err)
conf.SetNamespace(ns)
})

It("prepares `Immediate` storage class and virtual disk that use it", func() {
sc, err := GetDefaultStorageClass()
Expect(err).NotTo(HaveOccurred(), "cannot get default storage class\nstderr: %s", err)

defaultVolumeSnapshotClassName, err := GetVolumeSnapshotClassName(sc)
Expect(err).NotTo(HaveOccurred(), "cannot define default `VolumeSnapshotClass`\nstderr: %s", err)

if sc.Provisioner == LinstorProviderName {
storagePoolName := sc.Parameters["replicated.csi.storage.deckhouse.io/storagePool"]
storagePoolObj := sdsrepvolv1.ReplicatedStoragePool{}
err := GetObject(kc.ResourceReplicatedStoragePool, storagePoolName, &storagePoolObj, kc.GetOptions{})
Expect(err).NotTo(HaveOccurred(), "cannot get `storagePoolObj`: %s\nstderr: %s", storagePoolName, err)
Expect(storagePoolObj.Spec.Type).To(Equal(LVMThinName), "type of replicated storage pool should be `LVMThin`")
}

if *sc.VolumeBindingMode != storagev1.VolumeBindingImmediate {
immediateStorageClassName, err = CreateImmediateStorageClass(sc.Provisioner, testCaseLabel)
Expect(err).NotTo(HaveOccurred(), "%s", err)

virtualDisk := virtv2.VirtualDisk{}
vdFilePath := fmt.Sprintf("%s/vd/vd-alpine-http.yaml", conf.TestData.ImagesCreation)
err = UnmarshalResource(vdFilePath, &virtualDisk)
Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdFilePath, err)

virtualDisk.Spec.PersistentVolumeClaim.StorageClass = &immediateStorageClassName
err = WriteYamlObject(vdFilePath, &virtualDisk)
Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdFilePath, err)
} else {
immediateStorageClassName = sc.Name
}

virtualDiskSnapshot := virtv2.VirtualDiskSnapshot{}
vdSnapshotFilePath := fmt.Sprintf("%s/vdsnapshot/vdsnapshot.yaml", conf.TestData.ImagesCreation)
err = UnmarshalResource(vdSnapshotFilePath, &virtualDiskSnapshot)
Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdSnapshotFilePath, err)

virtualDiskSnapshot.Spec.VolumeSnapshotClassName = defaultVolumeSnapshotClassName
err = WriteYamlObject(vdSnapshotFilePath, &virtualDiskSnapshot)
Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdSnapshotFilePath, err)
})
})

Context("When resources are applied", func() {
It("result should be succeeded", func() {
res := kubectl.Apply(kc.ApplyOptions{
Expand Down Expand Up @@ -153,16 +128,6 @@ var _ = Describe("Virtual images creation", ginkgoutil.CommonE2ETestDecorators()
It("deletes test case resources", func() {
DeleteTestCaseResources(ResourcesToDelete{
KustomizationDir: conf.TestData.ImagesCreation,
AdditionalResources: []AdditionalResource{
{
Resource: kc.ResourceReplicatedStorageClass,
Labels: testCaseLabel,
},
{
Resource: kc.ResourceStorageClass,
Labels: testCaseLabel,
},
},
})
})
})
Expand Down
44 changes: 21 additions & 23 deletions tests/e2e/kubectl/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,25 @@ limitations under the License.
package kubectl

const (
ResourceNode Resource = "node"
ResourceNamespace Resource = "namespace"
ResourcePod Resource = "pod"
ResourceService Resource = "service"
ResourceStorageClass Resource = "storageclasses.storage.k8s.io"
ResourceVolumeSnapshotClass Resource = "volumesnapshotclasses.snapshot.storage.k8s.io"
ResourceProject Resource = "projects.deckhouse.io"
ResourceReplicatedStorageClass Resource = "replicatedstorageclasses.storage.deckhouse.io"
ResourceReplicatedStoragePool Resource = "replicatedstoragepools.storage.deckhouse.io"
ResourceKubevirtVM Resource = "internalvirtualizationvirtualmachines.internal.virtualization.deckhouse.io"
ResourceKubevirtVMI Resource = "internalvirtualizationvirtualmachineinstances.internal.virtualization.deckhouse.io"
ResourceKubevirtVMIM Resource = "internalvirtualizationvirtualmachineinstancemigrations.internal.virtualization.deckhouse.io"
ResourceModuleConfig Resource = "moduleconfigs.deckhouse.io"
ResourceVD Resource = "virtualdisks.virtualization.deckhouse.io"
ResourceVDSnapshot Resource = "virtualdisksnapshots.virtualization.deckhouse.io"
ResourceVM Resource = "virtualmachine.virtualization.deckhouse.io"
ResourceVMClass Resource = "virtualmachineclasses.virtualization.deckhouse.io"
ResourceVMIP Resource = "virtualmachineipaddresses.virtualization.deckhouse.io"
ResourceVMIPLease Resource = "virtualmachineipaddressleases.virtualization.deckhouse.io"
ResourceCVI Resource = "clustervirtualimages.virtualization.deckhouse.io"
ResourceVI Resource = "virtualimages.virtualization.deckhouse.io"
ResourceVMBDA Resource = "virtualmachineblockdeviceattachments.virtualization.deckhouse.io"
ResourceVMOP Resource = "virtualmachineoperations.virtualization.deckhouse.io"
ResourceNode Resource = "node"
ResourceNamespace Resource = "namespace"
ResourcePod Resource = "pod"
ResourceService Resource = "service"
ResourceStorageClass Resource = "storageclasses.storage.k8s.io"
ResourceVolumeSnapshotClass Resource = "volumesnapshotclasses.snapshot.storage.k8s.io"
ResourceProject Resource = "projects.deckhouse.io"
ResourceKubevirtVM Resource = "internalvirtualizationvirtualmachines.internal.virtualization.deckhouse.io"
ResourceKubevirtVMI Resource = "internalvirtualizationvirtualmachineinstances.internal.virtualization.deckhouse.io"
ResourceKubevirtVMIM Resource = "internalvirtualizationvirtualmachineinstancemigrations.internal.virtualization.deckhouse.io"
ResourceModuleConfig Resource = "moduleconfigs.deckhouse.io"
ResourceVD Resource = "virtualdisks.virtualization.deckhouse.io"
ResourceVDSnapshot Resource = "virtualdisksnapshots.virtualization.deckhouse.io"
ResourceVM Resource = "virtualmachine.virtualization.deckhouse.io"
ResourceVMClass Resource = "virtualmachineclasses.virtualization.deckhouse.io"
ResourceVMIP Resource = "virtualmachineipaddresses.virtualization.deckhouse.io"
ResourceVMIPLease Resource = "virtualmachineipaddressleases.virtualization.deckhouse.io"
ResourceCVI Resource = "clustervirtualimages.virtualization.deckhouse.io"
ResourceVI Resource = "virtualimages.virtualization.deckhouse.io"
ResourceVMBDA Resource = "virtualmachineblockdeviceattachments.virtualization.deckhouse.io"
ResourceVMOP Resource = "virtualmachineoperations.virtualization.deckhouse.io"
)
16 changes: 11 additions & 5 deletions tests/e2e/tests_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/cli-runtime/pkg/genericclioptions"
Expand Down Expand Up @@ -77,7 +76,6 @@ var (
d8Virtualization d8.D8Virtualization
git gt.Git
namePrefix string
defaultStorageClass *storagev1.StorageClass
phaseByVolumeBindingMode string
logStreamByV12nControllerPod = make(map[string]*el.LogStream, 0)
)
Expand All @@ -87,6 +85,10 @@ func init() {
if err != nil {
log.Fatal(err)
}
err = config.CheckStorageClassOption()
if err != nil {
log.Fatal(err)
}
err = config.CheckWithPostCleanUpOption()
if err != nil {
log.Fatal(err)
Expand Down Expand Up @@ -128,16 +130,20 @@ func init() {
if git, err = gt.NewGit(); err != nil {
log.Fatal(err)
}
if defaultStorageClass, err = GetDefaultStorageClass(); err != nil {
if conf.StorageClass.DefaultStorageClass, err = GetDefaultStorageClass(); err != nil {
log.Fatal(err)
}
if !config.SkipImmediateStorageClassCheck() {
if conf.StorageClass.ImmediateStorageClass, err = GetImmediateStorageClass(conf.StorageClass.DefaultStorageClass.Provisioner); err != nil {
log.Fatal(err)
}
}
if namePrefix, err = config.GetNamePrefix(); err != nil {
log.Fatal(err)
}
ChmodFile(conf.TestData.Sshkey, 0o600)
conf.Namespace = fmt.Sprintf("%s-%s", namePrefix, conf.Namespace)
conf.StorageClass.VolumeBindingMode = *defaultStorageClass.VolumeBindingMode
phaseByVolumeBindingMode = GetPhaseByVolumeBindingMode(conf)
phaseByVolumeBindingMode = GetPhaseByVolumeBindingMode(conf.StorageClass.DefaultStorageClass)
// TODO: get kustomization files from testdata directory when all tests will be refactored
var kustomizationFiles []string
v := reflect.ValueOf(conf.TestData)
Expand Down
35 changes: 27 additions & 8 deletions tests/e2e/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import (

virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition"
. "github.com/deckhouse/virtualization/tests/e2e/config"
"github.com/deckhouse/virtualization/tests/e2e/config"
"github.com/deckhouse/virtualization/tests/e2e/executor"
"github.com/deckhouse/virtualization/tests/e2e/helper"
kc "github.com/deckhouse/virtualization/tests/e2e/kubectl"
Expand Down Expand Up @@ -383,6 +383,25 @@ func GetDefaultStorageClass() (*storagev1.StorageClass, error) {
return defaultClasses[0], nil
}

func GetImmediateStorageClass(provisioner string) (*storagev1.StorageClass, error) {
scl := &storagev1.StorageClassList{}
err := GetObjects(kc.ResourceStorageClass, scl, kc.GetOptions{})
if err != nil {
return nil, err
}

for _, sc := range scl.Items {
if sc.Provisioner == provisioner && *sc.VolumeBindingMode == storagev1.VolumeBindingImmediate {
return &sc, nil
}
}

return nil, fmt.Errorf("immediate storage class does not found; please set up immediate storage class with the %q provisioner; to skip the immediate storage class check, set %s=yes",
provisioner,
config.SkipImmediateStorageClassCheckEnv,
)
}

func toIPNet(prefix netip.Prefix) *net.IPNet {
return &net.IPNet{
IP: prefix.Masked().Addr().AsSlice(),
Expand Down Expand Up @@ -491,14 +510,14 @@ func GetCondition(conditionType string, obj client.Object) (metav1.Condition, er
return metav1.Condition{}, fmt.Errorf("condition %s not found", conditionType)
}

func GetPhaseByVolumeBindingMode(c *Config) string {
switch c.StorageClass.VolumeBindingMode {
case "Immediate":
return PhaseReady
case "WaitForFirstConsumer":
return PhaseWaitForFirstConsumer
func GetPhaseByVolumeBindingMode(sc *storagev1.StorageClass) string {
switch *sc.VolumeBindingMode {
case storagev1.VolumeBindingImmediate:
return string(virtv2.DiskReady)
case storagev1.VolumeBindingWaitForFirstConsumer:
return string(virtv2.DiskWaitForFirstConsumer)
default:
return PhaseReady
return string(virtv2.DiskReady)
}
}

Expand Down
Loading
Loading