Skip to content

Commit b5ec810

Browse files
Merge pull request #375 from openebs/exclude_cordoned_node
test: add volume provisioning test on cordoned node
2 parents 0217261 + 26a721a commit b5ec810

File tree

5 files changed

+113
-15
lines changed

5 files changed

+113
-15
lines changed

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ require (
88
github.com/onsi/ginkgo v1.16.4
99
github.com/onsi/gomega v1.27.4
1010
github.com/openebs/google-analytics-4 v0.3.0
11-
github.com/openebs/lib-csi v0.8.2
11+
github.com/openebs/lib-csi v0.9.0
1212
github.com/pkg/errors v0.9.1
1313
github.com/prometheus/client_golang v1.15.1
1414
github.com/spf13/cobra v1.6.0

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -314,8 +314,8 @@ github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
314314
github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
315315
github.com/openebs/google-analytics-4 v0.3.0 h1:rAv9BC476ennyGPWAy4D3vU7TlxEKiynbLUP3VgeBUc=
316316
github.com/openebs/google-analytics-4 v0.3.0/go.mod h1:lKjvRs6HAYOlOTYjtOUp35iTutoXzRL0hOeVAvLAfZI=
317-
github.com/openebs/lib-csi v0.8.2 h1:HmoiZX3VXFPglwqnRPnRus7K58ixDWBa19OpPZGk2Ws=
318-
github.com/openebs/lib-csi v0.8.2/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc=
317+
github.com/openebs/lib-csi v0.9.0 h1:Yro5CN0cT3bRYOB1IdGPKXnBcPzxZnkxd4MLREAtFME=
318+
github.com/openebs/lib-csi v0.9.0/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc=
319319
github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
320320
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
321321
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/lvm_utils.go

-2
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ func vgEmpty(name string) bool {
7979
lvs, _, _ := execAtLocal("sudo", nil, args_lvs...)
8080
lvs_str := strings.TrimSpace(string(lvs))
8181
lv_cnt, _ := strconv.Atoi(lvs_str)
82-
fmt.Printf("lvs cnt is %d\n", lv_cnt)
8382
if lv_cnt != 0 {
8483
return false
8584
} else {
@@ -160,7 +159,6 @@ func removeVg(name string) {
160159
if current_retry < retries {
161160
vg_empty := vgEmpty(name)
162161
if vg_empty {
163-
fmt.Printf("No lv in vg before vg remove\n")
164162
break
165163
} else {
166164
fmt.Printf("lv in vg during retry %d\n", current_retry)

tests/provision_test.go

+17
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,22 @@ func vgPatternMatchPresentTest() {
149149
By("Deleting storage class", deleteStorageClass)
150150
}
151151

152+
func scheduleOnCordonedNodeTest() {
153+
device := setupVg(20, "lvmvg")
154+
defer cleanupVg(device, "lvmvg")
155+
By("Creating storage class", createStorageClass)
156+
By("Cordoning the node", cordonk8sNode)
157+
By("Creating and verifying PVC Bound status. It should not be Bound")
158+
createAndVerifyPVC(false)
159+
By("Uncordon the node", uncordonk8sNode)
160+
By("Verify the PVC gets Bound")
161+
verifyPVCStatus(pvcName, true)
162+
deleteAndVerifyPVC(pvcName)
163+
By("Verifying that PV doesnt exists after PVC deletion")
164+
verifyPVForPVC(false, pvcName)
165+
By("Deleting storage class", deleteStorageClass)
166+
}
167+
152168
func vgPatternNoMatchPresentTest() {
153169
device := setupVg(20, "lvmvg212")
154170
device_1 := setupVg(20, "lvmvg")
@@ -316,6 +332,7 @@ func schedulingTest() {
316332
By("###Running vg specified in sc not present test###", vgSpecifiedNotPresentTest)
317333
By("###Running lvmnode has vg matching vgpattern test###", vgPatternMatchPresentTest)
318334
By("###Running lvmnode doesnt have vg matching vgpattern test###", vgPatternNoMatchPresentTest)
335+
By("###Running volume schedule on Cordoned node test###", scheduleOnCordonedNodeTest)
319336
}
320337

321338
func capacityTest() {

tests/utils.go

+93-10
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package tests
33
import (
44
"context"
55
"fmt"
6+
"path/filepath"
67

78
"github.com/onsi/ginkgo"
89
"github.com/onsi/gomega"
@@ -12,6 +13,9 @@ import (
1213
corev1 "k8s.io/api/core/v1"
1314
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1415
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/client-go/kubernetes"
17+
"k8s.io/client-go/tools/clientcmd"
18+
"k8s.io/client-go/util/homedir"
1519

1620
"github.com/openebs/lvm-localpv/pkg/lvm"
1721
"github.com/openebs/lvm-localpv/tests/container"
@@ -249,14 +253,12 @@ func VerifyLVMVolume(expect_ready bool, expected_vg string) {
249253
// It gets deleted, by the csi provisioner only when the owner node of cr marks is
250254
// as Failed. So incase, we do a get of cr when the cr was being handled then we expect
251255
// state to be either Pending or Failed.
252-
fmt.Printf("checking vol object as vol is non nil, vol is %v\n", vol)
253256
gomega.Expect(vol.Status.State).To(gomega.Or(gomega.Equal("Pending"), gomega.Equal("Failed")),
254257
"While checking if lvmvolume: %s is in Pending or Failed state", pvcObj.Spec.VolumeName)
255258
}
256259
} else {
257260
gomega.Expect(err).To(gomega.BeNil(), "while fetching the lvm volume {%s}", pvcObj.Spec.VolumeName)
258261
if expected_vg != "" {
259-
fmt.Printf("vol is %v\n", vol)
260262
gomega.Expect(vol.Spec.VolGroup).To(gomega.Equal(expected_vg),
261263
"while checking volume group of lvm volume", pvcObj.Spec.VolumeName)
262264
} else {
@@ -302,14 +304,7 @@ func createAndVerifyPVC(expect_bound bool) {
302304
pvcName,
303305
OpenEBSNamespace,
304306
)
305-
ok := false
306-
if !expect_bound {
307-
ok = IsPVCPendingConsistently(pvcName)
308-
} else {
309-
ok = IsPVCBoundEventually(pvcName)
310-
}
311-
gomega.Expect(ok).To(gomega.Equal(true),
312-
"while checking the pvc status")
307+
verifyPVCStatus(pvcName, expect_bound)
313308

314309
pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{})
315310
gomega.Expect(err).To(
@@ -320,6 +315,18 @@ func createAndVerifyPVC(expect_bound bool) {
320315
)
321316
}
322317

318+
// Verifies state of already created pvc based on expect_bound.
319+
func verifyPVCStatus(pvc_name string, expect_bound bool) {
320+
ok := false
321+
if !expect_bound {
322+
ok = IsPVCPendingConsistently(pvc_name)
323+
} else {
324+
ok = IsPVCBoundEventually(pvc_name)
325+
}
326+
gomega.Expect(ok).To(gomega.Equal(true),
327+
"while checking the pvc status")
328+
}
329+
323330
func createAndVerifyBlockPVC(expect_bound bool) {
324331
var (
325332
err error
@@ -819,3 +826,79 @@ func createNodeDaemonSet(ds *appsv1.DaemonSet) {
819826
gomega.BeNil(),
820827
"creating node plugin daemonset %v", nodeDaemonSet)
821828
}
829+
830+
// Creates a k8s client using kubeconfig path.
831+
func getk8sClient() (client *kubernetes.Clientset) {
832+
kubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config")
833+
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
834+
gomega.Expect(err).To(
835+
gomega.BeNil(),
836+
"Could not created a k8s client",
837+
)
838+
client, c_err := kubernetes.NewForConfig(config)
839+
gomega.Expect(c_err).To(
840+
gomega.BeNil(),
841+
"Could not created a k8s client",
842+
)
843+
return client
844+
}
845+
846+
// Lists k8s nodes.
847+
func listNodes(client *kubernetes.Clientset) (nodes *corev1.NodeList) {
848+
nodes, n_err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
849+
gomega.Expect(n_err).To(
850+
gomega.BeNil(),
851+
"Could not list nodes",
852+
)
853+
return nodes
854+
}
855+
856+
// Cordons all k8s nodes.
857+
func cordonk8sNode() {
858+
client := getk8sClient()
859+
nodes := listNodes(client)
860+
for _, node := range nodes.Items {
861+
if !node.Spec.Unschedulable {
862+
c_err := cordonNode(client, &node)
863+
gomega.Expect(c_err).To(
864+
gomega.BeNil(),
865+
"Could not cordon node",
866+
)
867+
}
868+
}
869+
}
870+
871+
// UnCordons all k8s nodes.
872+
func uncordonk8sNode() {
873+
client := getk8sClient()
874+
nodes := listNodes(client)
875+
for _, node := range nodes.Items {
876+
if node.Spec.Unschedulable {
877+
c_err := uncordonNode(client, &node)
878+
gomega.Expect(c_err).To(
879+
gomega.BeNil(),
880+
"Could not uncordon node",
881+
)
882+
}
883+
}
884+
}
885+
886+
// Adds cordon taint to a specific node.
887+
func cordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error {
888+
updatedNode := node.DeepCopy()
889+
updatedNode.Spec.Unschedulable = true
890+
891+
_, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{})
892+
return err
893+
894+
}
895+
896+
// Removes cordon taint from a specific node.
897+
func uncordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error {
898+
updatedNode := node.DeepCopy()
899+
updatedNode.Spec.Unschedulable = false
900+
901+
_, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{})
902+
return err
903+
904+
}

0 commit comments

Comments
 (0)