test: add taints and tolerations e2e test (#5901)

Author: Devinwong

.github/renovate.json

@@ -278,12 +278,6 @@
         "gaopenghigh"
       ]
     },
-    {
-      "matchPackageNames": [
-        "containernetworking/azure-ipam"
-      ],
-      "groupName": "azure-ipam"
-    },
     {
       "matchPackageNames": [
         "oss/kubernetes/azure-cloud-node-manager"

e2e/kube.go

@@ -38,8 +38,8 @@ type Kubeclient struct {
 }
 
 const (
-	hostNetworkDebugAppLabel = "debug-mariner"
-	podNetworkDebugAppLabel  = "debugnonhost-mariner"
+	hostNetworkDebugAppLabel = "debug-mariner-tolerated"
+	podNetworkDebugAppLabel  = "debugnonhost-mariner-tolerated"
 )
 
 func getClusterKubeClient(ctx context.Context, resourceGroupName, clusterName string) (*Kubeclient, error) {
@@ -164,10 +164,6 @@ func (k *Kubeclient) WaitUntilNodeReady(ctx context.Context, t *testing.T, vmssN
 		node = castNode
 		nodeTaints, _ := json.Marshal(node.Spec.Taints)
 		nodeConditions, _ := json.Marshal(node.Status.Conditions)
-		if len(node.Spec.Taints) > 0 {
-			t.Logf("node %s is tainted. Taints: %s Conditions: %s", node.Name, string(nodeTaints), string(nodeConditions))
-			continue
-		}
 
 		for _, cond := range node.Status.Conditions {
 			if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue {
@@ -417,6 +413,23 @@ func daemonsetDebug(t *testing.T, deploymentName, targetNodeLabel, privateACRNam
 							},
 						},
 					},
+					// Set Tolerations to tolerate the node with test taints "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule".
+					// This is to ensure that the pod can be scheduled on the node with the taints.
+					// It won't affect other pods running on the same node.
+					Tolerations: []corev1.Toleration{
+						{
+							Key:      "testkey1",
+							Operator: corev1.TolerationOpEqual,
+							Value:    "value1",
+							Effect:   corev1.TaintEffectNoSchedule,
+						},
+						{
+							Key:      "testkey2",
+							Operator: corev1.TolerationOpEqual,
+							Value:    "value2",
+							Effect:   corev1.TaintEffectNoSchedule,
+						},
+					},
 				},
 			},
 		},
@@ -468,6 +481,23 @@ func podHTTPServerLinux(s *Scenario) *corev1.Pod {
 					},
 				},
 			},
+			// Set Tolerations to tolerate the node with test taints "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule".
+			// This is to ensure that the pod can be scheduled on the node with the taints.
+			// It won't affect other pods running on the same node.
+			Tolerations: []corev1.Toleration{
+				{
+					Key:      "testkey1",
+					Operator: corev1.TolerationOpEqual,
+					Value:    "value1",
+					Effect:   corev1.TaintEffectNoSchedule,
+				},
+				{
+					Key:      "testkey2",
+					Operator: corev1.TolerationOpEqual,
+					Value:    "value2",
+					Effect:   corev1.TaintEffectNoSchedule,
+				},
+			},
 			NodeSelector: map[string]string{
 				"kubernetes.io/hostname": s.Runtime.KubeNodeName,
 			},

e2e/scenario_test.go

@@ -824,18 +824,20 @@ func Test_Ubuntu2204_ArtifactStreaming_Scriptless(t *testing.T) {
 	})
 }
 
-func Test_Ubuntu2204_ChronyRestarts(t *testing.T) {
+func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) {
 	RunScenario(t, &Scenario{
-		Description: "Tests that the chrony service restarts if it is killed",
+		Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations",
 		Config: Config{
 			Cluster: ClusterKubenet,
 			VHD:     config.VHDUbuntu2204Gen2Containerd,
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.KubeletConfig["--register-with-taints"] = "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule"
 			},
 			Validator: func(ctx context.Context, s *Scenario) {
 				ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always")
 				ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "RestartSec=5")
 				ServiceCanRestartValidator(ctx, s, "chronyd", 10)
+				ValidateTaints(ctx, s)
 			},
 		},
 	})

e2e/validators.go

@@ -427,3 +427,19 @@ func GetFieldFromJsonObjectOnNode(ctx context.Context, s *Scenario, fileName str
 
 	return podExecResult.stdout.String()
 }
+
+// ValidateTaints checks if the node has the expected taints that are set in the kubelet config with --register-with-taints flag
+func ValidateTaints(ctx context.Context, s *Scenario) {
+	expectedTaints := s.Runtime.NBC.KubeletConfig["--register-with-taints"]
+	node, err := s.Runtime.Cluster.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.KubeNodeName, metav1.GetOptions{})
+	require.NoError(s.T, err, "failed to get node %q", s.Runtime.KubeNodeName)
+	actualTaints := ""
+	for i, taint := range node.Spec.Taints {
+		actualTaints += fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)
+		// add a comma if it's not the last element
+		if i < len(node.Spec.Taints)-1 {
+			actualTaints += ","
+		}
+	}
+	require.Equal(s.T, expectedTaints, actualTaints, "expected node %q to have taint %q, but got %q", s.Runtime.KubeNodeName, expectedTaints, actualTaints)
+}