From 21fd7467f4f8850d07f8aa2578d1618bcf722ca5 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Thu, 13 Feb 2025 11:38:37 +0530 Subject: [PATCH 01/10] refactor: extract cleanup fns & allow for cleanup in beforesuite --- pkg/test/integration/common/framework.go | 202 ++++++++++-------- .../machinecontroller/machine_test.go | 26 ++- 2 files changed, 141 insertions(+), 87 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index f71d0a615..9feed59c0 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -8,7 +8,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/gardener/machine-controller-manager/pkg/controller" "io" "log" "os" @@ -19,6 +18,8 @@ import ( "strings" "time" + "github.com/gardener/machine-controller-manager/pkg/controller" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -806,6 +807,12 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { } } + ginkgo.By("Cleaning any old resources") + if c.ControlCluster.McmClient != nil { + timeout := int64(900) + c.cleanTestResources(ctx, timeout) + } + ginkgo.By("Setup MachineClass") gomega.Expect(c.setupMachineClass()).To(gomega.BeNil()) @@ -826,19 +833,12 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { machineClass.SecretRef, machineClass.CredentialsSecretRef, ) - gomega.Expect(err). - NotTo( - gomega.HaveOccurred(), - ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By( - "Initializing orphan resource tracker", - ) + ginkgo.By("Initializing orphan resource tracker") err = c.resourcesTracker.InitializeResourcesTracker(machineClass, secretData, clusterName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) log.Println("orphan resource tracker initialized") - } // BeforeEachCheck checks if all the nodes are ready and the controllers are runnings @@ -1310,82 +1310,7 @@ func (c *IntegrationTestFramework) Cleanup() { if c.ControlCluster.McmClient != nil { timeout := int64(900) - // Check and delete machinedeployment resource - _, err := c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) - if err == nil { - log.Println("deleting test-machine-deployment") - watchMachinesDepl, _ := c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineDeployments(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineDeploymentObj.ResourceVersion - for event := range watchMachinesDepl.ResultChan() { - gomega.Expect(c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineDeployments(controlClusterNamespace). - Delete(ctx, "test-machine-deployment", metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) - if event.Type == watch.Deleted { - watchMachinesDepl.Stop() - log.Println("machinedeployment deleted") - } - } - } else { - log.Println(err.Error()) - } - // Check and delete machine resource - _, err = c.ControlCluster.McmClient. - MachineV1alpha1(). - Machines(controlClusterNamespace). - Get(ctx, "test-machine", metav1.GetOptions{}) - if err == nil { - log.Println("deleting test-machine") - watchMachines, _ := c.ControlCluster.McmClient. - MachineV1alpha1(). - Machines(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineObj.ResourceVersion - for event := range watchMachines.ResultChan() { - gomega.Expect(c.ControlCluster.McmClient. - MachineV1alpha1(). - Machines(controlClusterNamespace). - Delete(ctx, "test-machine", metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) - if event.Type == watch.Deleted { - watchMachines.Stop() - log.Println("machine deleted") - } - } - } else { - log.Println(err.Error()) - } - - for _, machineClassName := range testMachineClassResources { - // Check and delete machine class resource - _, err = c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineClasses(controlClusterNamespace). - Get(ctx, machineClassName, metav1.GetOptions{}) - if err == nil { - log.Printf("deleting %s machineclass", machineClassName) - watchMachineClass, _ := c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineClasses(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) - for event := range watchMachineClass.ResultChan() { - gomega.Expect(c.ControlCluster.McmClient. - MachineV1alpha1(). - MachineClasses(controlClusterNamespace). - Delete(ctx, machineClassName, metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) - if event.Type == watch.Deleted { - watchMachineClass.Stop() - log.Println("machineclass deleted") - } - } - } else { - log.Println(err.Error()) - } - } - + c.cleanTestResources(ctx, timeout) } if isControlSeed == "true" { // scale back up the MCM deployment to 1 in the Control Cluster @@ -1456,6 +1381,111 @@ func (c *IntegrationTestFramework) Cleanup() { } } + if mcsession.ExitCode() == -1 { + log.Println("killing mc process") + mcsession.Kill() + } + if mcmsession.ExitCode() == -1 { + log.Println("killing mcm process") + mcmsession.Kill() + } +} + +func (c *IntegrationTestFramework) cleanTestResources(ctx context.Context, timeout int64) { + // Check and delete machinedeployment resource + if err := c.cleanMachineDeployment(ctx, "test-machine-deployment", timeout); err != nil { + log.Println(err.Error()) + } + // Check and delete machine resource + if err := c.cleanMachine(ctx, "test-machine", timeout); err != nil { + log.Println(err.Error()) + } + + for _, machineClassName := range testMachineClassResources { + // Check and delete machine class resource + if err := c.cleanMachineClass(ctx, machineClassName, timeout); err != nil { + log.Println(err.Error()) + } + } +} + +func (c *IntegrationTestFramework) cleanMachineDeployment(ctx context.Context, name string, timeout int64) error { + _, err := c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineDeployments(controlClusterNamespace). + Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + log.Printf("deleting %s\n", name) + watchMachinesDepl, _ := c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineDeployments(controlClusterNamespace). + Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineDeploymentObj.ResourceVersion + for event := range watchMachinesDepl.ResultChan() { + gomega.Expect(c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineDeployments(controlClusterNamespace). + Delete(ctx, name, metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) + if event.Type == watch.Deleted { + watchMachinesDepl.Stop() + log.Println("machinedeployment deleted") + } + } + return nil +} + +func (c *IntegrationTestFramework) cleanMachine(ctx context.Context, name string, timeout int64) error { + _, err := c.ControlCluster.McmClient. + MachineV1alpha1(). + Machines(controlClusterNamespace). + Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + log.Printf("deleting %s\n", name) + watchMachines, _ := c.ControlCluster.McmClient. + MachineV1alpha1(). + Machines(controlClusterNamespace). + Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineObj.ResourceVersion + for event := range watchMachines.ResultChan() { + gomega.Expect(c.ControlCluster.McmClient. + MachineV1alpha1(). + Machines(controlClusterNamespace). + Delete(ctx, name, metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) + if event.Type == watch.Deleted { + watchMachines.Stop() + log.Println("machine deleted") + } + } + return nil +} + +func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machineClassName string, timeout int64) error { + _, err := c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineClasses(controlClusterNamespace). + Get(ctx, machineClassName, metav1.GetOptions{}) + if err != nil { + return err + } + + log.Printf("deleting %s machineclass", machineClassName) + watchMachineClass, _ := c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineClasses(controlClusterNamespace). + Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) + for event := range watchMachineClass.ResultChan() { + gomega.Expect(c.ControlCluster.McmClient. + MachineV1alpha1(). + MachineClasses(controlClusterNamespace). + Delete(ctx, machineClassName, metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) + if event.Type == watch.Deleted { + watchMachineClass.Stop() + log.Println("machineclass deleted") + } + } + return nil } func (c *IntegrationTestFramework) getNumberOfMachineSets(ctx context.Context, namespace string) int { diff --git a/pkg/util/provider/machinecontroller/machine_test.go b/pkg/util/provider/machinecontroller/machine_test.go index a658641f4..493988e87 100644 --- a/pkg/util/provider/machinecontroller/machine_test.go +++ b/pkg/util/provider/machinecontroller/machine_test.go @@ -226,7 +226,7 @@ var _ = Describe("machine", func() { action: machineapi.Machine{ Spec: machineapi.MachineSpec{ Class: machineapi.ClassSpec{ - Kind: "AWSMachineClass", + Kind: "MachineClass", Name: "aws", }, }, @@ -234,6 +234,30 @@ var _ = Describe("machine", func() { expect: field.ErrorList{}, }), ) + DescribeTable("#machine validation fails with no class name", + func(data *data) { + errList := validation.ValidateMachine(&data.action) + Expect(errList).To(Equal(data.expect)) + }, + Entry("aws", &data{ + action: machineapi.Machine{ + Spec: machineapi.MachineSpec{ + Class: machineapi.ClassSpec{ + Kind: "MachineClass", + Name: "", + }, + }, + }, + expect: field.ErrorList{ + { + Type: "FieldValueRequired", + Field: "spec.class.name", + BadValue: "", + Detail: "Name is required", + }, + }, + }), + ) }) Describe("#ValidateMachineClass", func() { From 466365453089e5c51055d50dc54afa9b22c22f25 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Sat, 15 Feb 2025 02:15:00 +0530 Subject: [PATCH 02/10] fix: kill processes run by IT before and after running the suite Iterates through a list of known processes for MC and MCM started by the test suite and ensures they are terminated by calling `Kill()` on them to prevent un-necessary failures due to "socket being used" by the leftover processes. --- pkg/test/integration/common/framework.go | 101 +++++++++++++++++++++-- 1 file changed, 92 insertions(+), 9 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 9feed59c0..2e69cd0fe 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -5,6 +5,8 @@ package common import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" @@ -12,6 +14,7 @@ import ( "log" "os" "os/exec" + "path" "path/filepath" "regexp" "strconv" @@ -99,6 +102,9 @@ var ( //values for gardener-node-agent-secret-name gnaSecretNameLabelValue = os.Getenv("GNA_SECRET_NAME") + + // Should the CR be preserved during cleanup at the end + preserveCRDuringCleanup = os.Getenv("PRESERVE_CRD_AT_END") ) // ProviderSpecPatch struct holds tags for provider, which we want to patch the machineclass with @@ -772,6 +778,9 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { ginkgo.By("Checking for the clusters if provided are available") gomega.Expect(c.initalizeClusters()).To(gomega.BeNil()) + ginkgo.By("Killing any existing processes") + // FIXME: errHandling + _ = stopMCM(ctx) //setting up MCM either locally or by deploying after checking conditions if isControlSeed == "true" { @@ -1308,7 +1317,8 @@ func (c *IntegrationTestFramework) Cleanup() { } } - if c.ControlCluster.McmClient != nil { + log.Printf("Preserve CRD: %s\n", preserveCRDuringCleanup) + if c.ControlCluster.McmClient != nil && preserveCRDuringCleanup != "true" { timeout := int64(900) c.cleanTestResources(ctx, timeout) } @@ -1381,14 +1391,9 @@ func (c *IntegrationTestFramework) Cleanup() { } } - if mcsession.ExitCode() == -1 { - log.Println("killing mc process") - mcsession.Kill() - } - if mcmsession.ExitCode() == -1 { - log.Println("killing mcm process") - mcmsession.Kill() - } + ginkgo.By("Killing any existing processes") + // FIXME: errHandling + _ = stopMCM(ctx) } func (c *IntegrationTestFramework) cleanTestResources(ctx context.Context, timeout int64) { @@ -1488,6 +1493,84 @@ func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machin return nil } +func stopMCM(ctx context.Context) (err error) { + processesToKill := []string{ + "controller_manager --control-kubeconfig", + "main --control-kubeconfig", + "go run cmd/machine-controller/main.go", + "go run cmd/machine-controller-manager/controller_manager.go", + "make --directory=../../..", + } + for _, proc := range processesToKill { + pids, err := findAndKillProcess(ctx, proc) + if err != nil { + return err + } + if len(pids) > 0 { + log.Printf("stopMCM killed MCM process(es) with pid(s): %v\n", pids) + } + } + return +} + +func findAndKillProcess(ctx context.Context, prefix string) (pids []int, err error) { + pids, err = findPidsByPrefix(ctx, prefix) + if len(pids) == 0 { + err = fmt.Errorf("failed to find pid(s) for process with prefix %q", prefix) + return + } + var proc *os.Process + for _, pid := range pids { + proc, err = os.FindProcess(pid) + if err != nil { + err = fmt.Errorf("failed to find process with PID %d: %v", pid, err) + return + } + err = proc.Kill() + if err != nil { + err = fmt.Errorf("failed to kill process with PID %d: %v", pid, err) + return + } + } + return +} + +func findPidsByPrefix(ctx context.Context, prefix string) (pids []int, err error) { + cmd := exec.CommandContext(ctx, "ps", "-e", "-o", "pid,command") + psOutput, err := cmd.Output() + if err != nil { + log.Printf("FindProcess could not run ps command: %v", err) + return + } + scanner := bufio.NewScanner(bytes.NewReader(psOutput)) + var pid int + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + // Process the PID and command columns + pidStr := fields[0] + commandPath := fields[1] + commandName := path.Base(commandPath) + if len(fields) > 2 { + commandName = commandName + " " + strings.Join(fields[2:], " ") + } + + if strings.HasPrefix(commandName, prefix) { + log.Println(commandName) + pid, err = strconv.Atoi(pidStr) + if err != nil { + err = fmt.Errorf("invalid pid: %s", pidStr) + return + } + pids = append(pids, pid) + } + } + return +} + func (c *IntegrationTestFramework) getNumberOfMachineSets(ctx context.Context, namespace string) int { machineSets, err := c.ControlCluster.McmClient.MachineV1alpha1().MachineSets(namespace).List(ctx, metav1.ListOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) From 830ab9774fcb0db6de7f6f03e46c1351f3fbece5 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 17 Feb 2025 23:17:28 +0530 Subject: [PATCH 03/10] cleanup: remove CONTAINER_IMAGE checks and related fn --- pkg/test/integration/common/framework.go | 337 ++--------------------- 1 file changed, 22 insertions(+), 315 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 2e69cd0fe..285795c93 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -234,226 +234,6 @@ func (c *IntegrationTestFramework) initalizeClusters() error { return nil } -func (c *IntegrationTestFramework) prepareMcmDeployment( - - mcContainerImage string, - mcmContainerImage string, - byCreating bool) error { - - ctx := context.Background() - - if len(machineControllerManagerDeploymentName) == 0 { - machineControllerManagerDeploymentName = "machine-controller-manager" - } - - if byCreating { - // Create clusterroles and clusterrolebindings for control and target cluster - // Create secret containing target kubeconfig file - // Create machine-deployment using the yaml file - controlClusterRegexp, _ := regexp.Compile("control-cluster-role") - targetClusterRegexp, _ := regexp.Compile("target-cluster-role") - log.Printf("Creating required roles and rolebinginds") - - err := filepath.Walk( - filepath.Join( - mcmRepoPath, - "kubernetes/deployment/out-of-tree/", - ), - func( - path string, - info os.FileInfo, - err error, - ) error { - - if err != nil { - return err - } - - if !info.IsDir() { - if controlClusterRegexp.MatchString(info.Name()) { - err = c. - ControlCluster. - ApplyFiles( - path, - controlClusterNamespace, - ) - } else if targetClusterRegexp.MatchString(info.Name()) { - err = c. - TargetCluster. - ApplyFiles( - path, - "default", - ) - } - } - return err - }) - - if err != nil { - return err - } - - configFile, _ := os.ReadFile(c.TargetCluster.KubeConfigFilePath) - if _, err := c.ControlCluster.Clientset. - CoreV1(). - Secrets(controlClusterNamespace). - Create(ctx, - &coreV1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-controller-manager", - }, - Data: map[string][]byte{ - "kubeconfig": configFile, - }, - Type: coreV1.SecretTypeOpaque, - }, - metav1.CreateOptions{}, - ); err != nil { - return err - } - - if err := c.ControlCluster. - ApplyFiles( - "../../../kubernetes/deployment.yaml", - controlClusterNamespace, - ); err != nil { - return err - } - } - - // mcmDeploymentOrigObj holds a copy of original mcm deployment - result, getErr := c.ControlCluster.Clientset. - AppsV1(). - Deployments(controlClusterNamespace). - Get(ctx, machineControllerManagerDeploymentName, metav1.GetOptions{}) - - if getErr != nil { - log.Printf("failed to get latest version of Deployment: %v", getErr) - return getErr - } - - mcmDeploymentOrigObj = result - - // update containers spec - providerSpecificRegexp, _ := regexp.Compile(mcContainerPrefix) - - containers := mcmDeploymentOrigObj.Spec.Template.Spec.Containers - - for i := range containers { - if providerSpecificRegexp.MatchString(containers[i].Image) { - // set container image to mcContainerImageTag as the name of the container contains provider - if len(mcContainerImage) != 0 { - containers[i].Image = mcContainerImage - var isOptionAvailable bool - for option := range containers[i].Command { - if strings.Contains(containers[i].Command[option], "machine-drain-timeout=") { - isOptionAvailable = true - containers[i].Command[option] = "--machine-drain-timeout=5m" - } - } - if !isOptionAvailable { - containers[i].Command = append(containers[i].Command, "--machine-drain-timeout=5m") - } - } - } else { - // set container image to mcmContainerImageTag as the name of container contains provider - if len(mcmContainerImage) != 0 { - containers[i].Image = mcmContainerImage - } - - // set machine-safety-overshooting-period to 300ms for freeze check to succeed - var isOptionAvailable bool - for option := range containers[i].Command { - if strings.Contains(containers[i].Command[option], "machine-safety-overshooting-period=") { - isOptionAvailable = true - containers[i].Command[option] = "--machine-safety-overshooting-period=300ms" - } - } - if !isOptionAvailable { - containers[i].Command = append(containers[i].Command, "--machine-safety-overshooting-period=300ms") - } - } - } - - // apply updated containers spec to mcmDeploymentObj in kubernetes cluster - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Retrieve the latest version of Deployment before attempting to update - // RetryOnConflict uses exponential backoff to avoid exhausting the apiserver - - mcmDeployment, getErr := c.ControlCluster.Clientset. - AppsV1(). - Deployments(controlClusterNamespace). - Get(ctx, - machineControllerManagerDeploymentName, - metav1.GetOptions{}, - ) - - if getErr != nil { - log.Printf("failed to get latest version of Deployment: %v", getErr) - return getErr - } - - mcmDeployment.Spec.Template.Spec.Containers = containers - _, updateErr := c.ControlCluster.Clientset. - AppsV1(). - Deployments(controlClusterNamespace). - Update(ctx, mcmDeployment, metav1.UpdateOptions{}) - return updateErr - }) - - ginkgo.By("Checking controllers are ready in kubernetes cluster") - gomega.Eventually( - func() error { - deployment, err := c.ControlCluster.Clientset. - AppsV1(). - Deployments(controlClusterNamespace). - Get(ctx, - machineControllerManagerDeploymentName, - metav1.GetOptions{}, - ) - if err != nil { - return err - } - if deployment.Status.ReadyReplicas == 1 { - pods, err := c.ControlCluster.Clientset. - CoreV1(). - Pods(controlClusterNamespace). - List( - ctx, - metav1.ListOptions{ - LabelSelector: "role=machine-controller-manager", - }, - ) - if err != nil { - return err - } - podsCount := len(pods.Items) - readyPods := 0 - for _, pod := range pods.Items { - if len(pod.Status.ContainerStatuses) < 2 { - return fmt.Errorf("containers(s) not ready") - } - if !pod.Status.ContainerStatuses[0].Ready { - return fmt.Errorf("containers(s) not ready.\n%s", pod.Status.ContainerStatuses[0].State.String()) - } - if !pod.Status.ContainerStatuses[1].Ready { - return fmt.Errorf("container(s) not ready.\n%s", pod.Status.ContainerStatuses[1].State.String()) - } - readyPods++ - } - if podsCount == readyPods { - return nil - } - } - return fmt.Errorf("deployment replicas are not ready.\n%s", deployment.Status.String()) - }, - c.timeout, - c.pollingInterval). - Should(gomega.BeNil()) - - return retryErr -} - func (c *IntegrationTestFramework) scaleMcmDeployment(replicas int32) error { ctx := context.Background() @@ -772,34 +552,22 @@ func (c *IntegrationTestFramework) runControllersLocally() { func (c *IntegrationTestFramework) SetupBeforeSuite() { ctx := context.Background() log.SetOutput(ginkgo.GinkgoWriter) - mcContainerImage := os.Getenv("MC_CONTAINER_IMAGE") - mcmContainerImage := os.Getenv("MCM_CONTAINER_IMAGE") ginkgo.By("Checking for the clusters if provided are available") gomega.Expect(c.initalizeClusters()).To(gomega.BeNil()) ginkgo.By("Killing any existing processes") - // FIXME: errHandling - _ = stopMCM(ctx) + stopMCM(ctx) //setting up MCM either locally or by deploying after checking conditions - if isControlSeed == "true" { - - if len(mcContainerImage) != 0 || len(mcmContainerImage) != 0 { - ginkgo.By("Updating MCM Deployemnt") - gomega.Expect(c.prepareMcmDeployment(mcContainerImage, mcmContainerImage, false)).To(gomega.BeNil()) - } else { - checkMcmRepoAvailable() - ginkgo.By("Scaledown existing machine controllers") - gomega.Expect(c.scaleMcmDeployment(0)).To(gomega.BeNil()) + checkMcmRepoAvailable() - c.runControllersLocally() - } + if isControlSeed == "true" { + ginkgo.By("Scaledown existing machine controllers") + gomega.Expect(c.scaleMcmDeployment(0)).To(gomega.BeNil()) } else { //TODO : Scaledown the MCM deployment of the actual seed of the target cluster - checkMcmRepoAvailable() - //create the custom resources in the control cluster using yaml files //available in kubernetes/crds directory of machine-controller-manager repo //resources to be applied are machineclass, machines, machinesets and machinedeployment @@ -807,14 +575,8 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { gomega.Expect(c.ControlCluster. ApplyFiles( filepath.Join(mcmRepoPath, "kubernetes/crds"), controlClusterNamespace)).To(gomega.BeNil()) - - if len(mcContainerImage) != 0 || len(mcmContainerImage) != 0 { - ginkgo.By("Creating MCM Deployemnt") - gomega.Expect(c.prepareMcmDeployment(mcContainerImage, mcmContainerImage, true)).To(gomega.BeNil()) - } else { - c.runControllersLocally() - } } + c.runControllersLocally() ginkgo.By("Cleaning any old resources") if c.ControlCluster.McmClient != nil { @@ -1338,62 +1100,10 @@ func (c *IntegrationTestFramework) Cleanup() { ); err != nil { log.Printf("Error occured while deleting crds. %s", err.Error()) } - - if len(os.Getenv("MC_CONTAINER_IMAGE")) != 0 || len(os.Getenv("MCM_CONTAINER_IMAGE")) != 0 { - log.Println("Deleting Clusterroles") - - if err := c.ControlCluster.RbacClient.ClusterRoles().Delete( - ctx, - "machine-controller-manager-control", - metav1.DeleteOptions{}, - ); err != nil { - log.Printf("Error occured while deleting clusterrole. %s", err.Error()) - } - - if err := c.ControlCluster.RbacClient.ClusterRoleBindings().Delete( - ctx, - "machine-controller-manager-control", - metav1.DeleteOptions{}, - ); err != nil { - log.Printf("Error occured while deleting clusterrolebinding . %s", err.Error()) - } - - if err := c.TargetCluster.RbacClient.ClusterRoles().Delete( - ctx, - "machine-controller-manager-target", - metav1.DeleteOptions{}, - ); err != nil { - log.Printf("Error occured while deleting clusterrole . %s", err.Error()) - } - - if err := c.TargetCluster.RbacClient.ClusterRoleBindings().Delete( - ctx, - "machine-controller-manager-target", - metav1.DeleteOptions{}, - ); err != nil { - log.Printf("Error occured while deleting clusterrolebinding . %s", err.Error()) - } - - log.Println("Deleting MCM deployment") - - gomega.Expect(c.ControlCluster.Clientset. - CoreV1(). - Secrets(controlClusterNamespace). - Delete(ctx, - "machine-controller-manager", - metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) - gomega.Expect(c.ControlCluster.Clientset. - AppsV1(). - Deployments(controlClusterNamespace). - Delete(ctx, - machineControllerManagerDeploymentName, - metav1.DeleteOptions{})).To(gomega.Or(gomega.Succeed(), matchers.BeNotFoundError())) - } } ginkgo.By("Killing any existing processes") - // FIXME: errHandling - _ = stopMCM(ctx) + stopMCM(ctx) } func (c *IntegrationTestFramework) cleanTestResources(ctx context.Context, timeout int64) { @@ -1493,7 +1203,7 @@ func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machin return nil } -func stopMCM(ctx context.Context) (err error) { +func stopMCM(ctx context.Context) { processesToKill := []string{ "controller_manager --control-kubeconfig", "main --control-kubeconfig", @@ -1504,32 +1214,29 @@ func stopMCM(ctx context.Context) (err error) { for _, proc := range processesToKill { pids, err := findAndKillProcess(ctx, proc) if err != nil { - return err + log.Println(err.Error()) } if len(pids) > 0 { log.Printf("stopMCM killed MCM process(es) with pid(s): %v\n", pids) } } - return } func findAndKillProcess(ctx context.Context, prefix string) (pids []int, err error) { pids, err = findPidsByPrefix(ctx, prefix) - if len(pids) == 0 { - err = fmt.Errorf("failed to find pid(s) for process with prefix %q", prefix) - return - } - var proc *os.Process - for _, pid := range pids { - proc, err = os.FindProcess(pid) - if err != nil { - err = fmt.Errorf("failed to find process with PID %d: %v", pid, err) - return - } - err = proc.Kill() - if err != nil { - err = fmt.Errorf("failed to kill process with PID %d: %v", pid, err) - return + if len(pids) != 0 { + var proc *os.Process + for _, pid := range pids { + proc, err = os.FindProcess(pid) + if err != nil { + err = fmt.Errorf("failed to find process with PID %d: %v", pid, err) + return + } + err = proc.Kill() + if err != nil { + err = fmt.Errorf("failed to kill process with PID %d: %v", pid, err) + return + } } } return From 663c42103aadfcb72d8be8f1d120f4b8e8278d82 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Fri, 21 Feb 2025 15:26:27 +0530 Subject: [PATCH 04/10] refactor: added event based machineset freeze check instead of logs --- pkg/controller/machine_safety.go | 2 + pkg/test/integration/common/framework.go | 105 +++++++---------------- 2 files changed, 35 insertions(+), 72 deletions(-) diff --git a/pkg/controller/machine_safety.go b/pkg/controller/machine_safety.go index ffe330001..1d5d8474a 100644 --- a/pkg/controller/machine_safety.go +++ b/pkg/controller/machine_safety.go @@ -330,6 +330,7 @@ func (c *controller) freezeMachineSetAndDeployment(ctx context.Context, machineS } } + c.recorder.Eventf(machineSet, "Normal", "FrozeMachineSet", "SafetyController: Froze MachineSet %s due to replica overshooting", machineSet.Name) klog.V(2).Infof("SafetyController: Froze MachineSet %q due to overshooting of replicas", machineSet.Name) return nil } @@ -398,6 +399,7 @@ func (c *controller) unfreezeMachineSet(ctx context.Context, machineSet *v1alpha return err } + c.recorder.Eventf(machineSet, "Normal", "UnfrozeMachineSet", "SafetyController: Unfroze MachineSet %s", machineSet.Name) klog.V(2).Infof("SafetyController: Unfroze MachineSet %q", machineSet.Name) return nil } diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 285795c93..12366fdfe 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -10,13 +10,11 @@ import ( "context" "encoding/json" "fmt" - "io" "log" "os" "os/exec" "path" "path/filepath" - "regexp" "strconv" "strings" "time" @@ -28,7 +26,6 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" appsV1 "k8s.io/api/apps/v1" - coreV1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" @@ -728,6 +725,7 @@ func (c *IntegrationTestFramework) ControllerTests() { // Testcase #02 | machine deployment ginkgo.Describe("machine deployment resource", func() { var initialNodes int16 // initialization should be part of creation test logic + initialEventCount := 0 ginkgo.Context("creation with replicas=0, scale up with replicas=1", func() { ginkgo.It("should not lead to errors and add 1 more node to target cluster", func() { var machineDeployment *v1alpha1.MachineDeployment @@ -738,7 +736,7 @@ func (c *IntegrationTestFramework) ControllerTests() { gomega.Expect(c.ControlCluster.CreateMachineDeployment(controlClusterNamespace, gnaSecretNameLabelValue, 0)).To(gomega.BeNil()) ginkgo.By("Waiting for Machine Set to be created") - gomega.Eventually(func() int { return c.getNumberOfMachineSets(ctx, controlClusterNamespace) }, c.timeout, c.pollingInterval).Should(gomega.BeNumerically("==", 1)) + gomega.Eventually(func() int { return len(c.getTestMachineSets(ctx, controlClusterNamespace)) }, c.timeout, c.pollingInterval).Should(gomega.BeNumerically("==", 1)) ginkgo.By("Updating machineDeployment replicas to 1") retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -792,6 +790,8 @@ func (c *IntegrationTestFramework) ControllerTests() { c.timeout, c.pollingInterval). Should(gomega.BeNumerically("==", initialNodes+1)) + ginkgo.By("Fetching initial number of machineset freeze events") + initialEventCount = c.machineSetFreezeEventCount(ctx, controlClusterNamespace) }) }) ginkgo.Context("scale-up with replicas=6", func() { @@ -855,68 +855,9 @@ func (c *IntegrationTestFramework) ControllerTests() { }) // rapid scaling back to 2, should lead to freezing and unfreezing ginkgo.It("should freeze and unfreeze machineset temporarily", func() { - if mcsession == nil { - // controllers running in pod - // Create log file from container log - mcmOutputFile, err := rotateOrAppendLogFile(mcmLogFile, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - mcOutputFile, err := rotateOrAppendLogFile(mcLogFile, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Reading container log is leading to no errors") - podList, err := c.ControlCluster.Clientset. - CoreV1(). - Pods(controlClusterNamespace). - List( - ctx, - metav1.ListOptions{ - LabelSelector: "role=machine-controller-manager", - }, - ) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - mcmPod := podList.Items[0] - providerSpecificRegexp, _ := regexp.Compile(mcContainerPrefix) - containers := mcmPod.Spec.Containers - - for i := range containers { - if providerSpecificRegexp.Match([]byte(containers[i].Image)) { - readCloser, err := c.ControlCluster.Clientset.CoreV1(). - Pods(controlClusterNamespace). - GetLogs( - mcmPod.Name, - &coreV1.PodLogOptions{ - Container: containers[i].Name, - }, - ).Stream(ctx) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = io.Copy(mcOutputFile, readCloser) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - readCloser, err := c.ControlCluster.Clientset.CoreV1(). - Pods(controlClusterNamespace). - GetLogs(mcmPod.Name, &coreV1.PodLogOptions{ - Container: containers[i].Name, - }).Stream(ctx) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = io.Copy(mcmOutputFile, readCloser) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - } - - ginkgo.By("Searching for Froze in mcm log file") - frozeRegexp, _ := regexp.Compile(` Froze MachineSet`) - gomega.Eventually(func() bool { - data, _ := os.ReadFile(mcmLogFile) // #nosec G304 -- Test only - return frozeRegexp.Match(data) - }, c.timeout, c.pollingInterval).Should(gomega.BeTrue()) - - ginkgo.By("Searching Unfroze in mcm log file") - unfrozeRegexp, _ := regexp.Compile(` Unfroze MachineSet`) - gomega.Eventually(func() bool { - data, _ := os.ReadFile(mcmLogFile) // #nosec G304 -- Test only - return unfrozeRegexp.Match(data) - }, c.timeout, c.pollingInterval).Should(gomega.BeTrue()) + gomega.Eventually(func() int { + return c.machineSetFreezeEventCount(ctx, controlClusterNamespace) + }).Should(gomega.Equal(initialEventCount + 2)) }) }) ginkgo.Context("updation to v2 machine-class and replicas=4", func() { @@ -1079,8 +1020,9 @@ func (c *IntegrationTestFramework) Cleanup() { } } - log.Printf("Preserve CRD: %s\n", preserveCRDuringCleanup) - if c.ControlCluster.McmClient != nil && preserveCRDuringCleanup != "true" { + if preserveCRDuringCleanup == "true" { + log.Printf("Preserve CRD: %s\n", preserveCRDuringCleanup) + } else if c.ControlCluster.McmClient != nil { timeout := int64(900) c.cleanTestResources(ctx, timeout) } @@ -1278,16 +1220,35 @@ func findPidsByPrefix(ctx context.Context, prefix string) (pids []int, err error return } -func (c *IntegrationTestFramework) getNumberOfMachineSets(ctx context.Context, namespace string) int { +func (c *IntegrationTestFramework) getTestMachineSets(ctx context.Context, namespace string) []string { machineSets, err := c.ControlCluster.McmClient.MachineV1alpha1().MachineSets(namespace).List(ctx, metav1.ListOptions{}) + testMachineSets := []string{} gomega.Expect(err).NotTo(gomega.HaveOccurred()) - count := 0 for _, machineSet := range machineSets.Items { if machineSet.OwnerReferences[0].Name == "test-machine-deployment" { - count++ + testMachineSets = append(testMachineSets, machineSet.Name) + } + } + return testMachineSets +} + +func (c *IntegrationTestFramework) machineSetFreezeEventCount(ctx context.Context, namespace string) int { + eventCount := 0 + testMachineSets := c.getTestMachineSets(ctx, namespace) + for _, machineSet := range testMachineSets { + for _, reason := range []string{"reason=FrozeMachineSet", "reason=UnfrozeMachineSet"} { + event := fmt.Sprintf("%s,involvedObject.name=%s", reason, machineSet) + // TODO: handle error + frozenEvents, _ := c.ControlCluster.Clientset. + CoreV1(). + Events(namespace). + List(ctx, metav1.ListOptions{ + FieldSelector: event, + }) + eventCount += len(frozenEvents.Items) } } - return count + return eventCount } func checkMcmRepoAvailable() { From f3ae69f37ff69d6c2433af3a867bb4c7d23d1e9e Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Thu, 27 Feb 2025 16:50:56 +0530 Subject: [PATCH 05/10] framework: add changes to support virtual provider --- pkg/test/integration/common/framework.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 12366fdfe..6a352f12a 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -102,6 +102,9 @@ var ( // Should the CR be preserved during cleanup at the end preserveCRDuringCleanup = os.Getenv("PRESERVE_CRD_AT_END") + + // Are the tests running for the virtual provider + isVirtualProvider = os.Getenv("IS_VIRTUAL_PROVIDER") ) // ProviderSpecPatch struct holds tags for provider, which we want to patch the machineclass with @@ -385,6 +388,7 @@ func (c *IntegrationTestFramework) setupMachineClass() error { Labels: machineClasses.Items[0].ObjectMeta.Labels, Annotations: machineClasses.Items[0].ObjectMeta.Annotations, }, + NodeTemplate: machineClasses.Items[0].NodeTemplate, ProviderSpec: machineClasses.Items[0].ProviderSpec, SecretRef: machineClasses.Items[0].SecretRef, CredentialsSecretRef: machineClasses.Items[0].CredentialsSecretRef, @@ -559,10 +563,10 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { checkMcmRepoAvailable() - if isControlSeed == "true" { + if isControlSeed == "true" && isVirtualProvider != "true" { ginkgo.By("Scaledown existing machine controllers") gomega.Expect(c.scaleMcmDeployment(0)).To(gomega.BeNil()) - } else { + } else if isControlSeed != "true" { //TODO : Scaledown the MCM deployment of the actual seed of the target cluster //create the custom resources in the control cluster using yaml files @@ -573,6 +577,7 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { ApplyFiles( filepath.Join(mcmRepoPath, "kubernetes/crds"), controlClusterNamespace)).To(gomega.BeNil()) } + c.runControllersLocally() ginkgo.By("Cleaning any old resources") @@ -638,6 +643,12 @@ func (c *IntegrationTestFramework) ControllerTests() { var initialNodes int16 ginkgo.Context("creation", func() { ginkgo.It("should not lead to any errors and add 1 more node in target cluster", func() { + // In case of existing deployments creating nodes when starting virtual + // provider, the change in node count can be >1, this delay prevents + // checking node count immediately to allow for a correct initial count + if isVirtualProvider == "true" { + time.Sleep(2 * time.Second) + } // Probe nodes currently available in target cluster initialNodes = c.TargetCluster.GetNumberOfNodes() ginkgo.By("Checking for errors") @@ -1147,6 +1158,8 @@ func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machin func stopMCM(ctx context.Context) { processesToKill := []string{ + "machine-controller-manager --control-kubeconfig", // virt + "machine-controller --control-kubeconfig", // virt "controller_manager --control-kubeconfig", "main --control-kubeconfig", "go run cmd/machine-controller/main.go", From d610c1d8d91f2e8dba7d3ddb298fa7b57744e44e Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 3 Mar 2025 08:49:37 +0530 Subject: [PATCH 06/10] Address review comments --- pkg/test/integration/common/framework.go | 161 ++++++++++++----------- 1 file changed, 86 insertions(+), 75 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 6a352f12a..36e8d0f68 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -38,6 +38,8 @@ import ( const ( dwdIgnoreScalingAnnotation = "dependency-watchdog.gardener.cloud/ignore-scaling" + testMachineDeploymentName = "test-machine-deployment" + testMachineName = "test-machine" ) var ( @@ -100,7 +102,7 @@ var ( //values for gardener-node-agent-secret-name gnaSecretNameLabelValue = os.Getenv("GNA_SECRET_NAME") - // Should the CR be preserved during cleanup at the end + // Specifies whether the CRDs should be preserved during cleanup at the end preserveCRDuringCleanup = os.Getenv("PRESERVE_CRD_AT_END") // Are the tests running for the virtual provider @@ -388,7 +390,7 @@ func (c *IntegrationTestFramework) setupMachineClass() error { Labels: machineClasses.Items[0].ObjectMeta.Labels, Annotations: machineClasses.Items[0].ObjectMeta.Annotations, }, - NodeTemplate: machineClasses.Items[0].NodeTemplate, + NodeTemplate: machineClasses.Items[0].NodeTemplate, // virtual ProviderSpec: machineClasses.Items[0].ProviderSpec, SecretRef: machineClasses.Items[0].SecretRef, CredentialsSecretRef: machineClasses.Items[0].CredentialsSecretRef, @@ -617,12 +619,10 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { // BeforeEachCheck checks if all the nodes are ready and the controllers are runnings func (c *IntegrationTestFramework) BeforeEachCheck() { ginkgo.BeforeEach(func() { - if len(os.Getenv("MC_CONTAINER_IMAGE")) == 0 && len(os.Getenv("MCM_CONTAINER_IMAGE")) == 0 { - ginkgo.By("Checking machineController process is running") - gomega.Expect(mcsession.ExitCode()).Should(gomega.Equal(-1)) - ginkgo.By("Checking machineControllerManager process is running") - gomega.Expect(mcmsession.ExitCode()).Should(gomega.Equal(-1)) - } + ginkgo.By("Checking machineController process is running") + gomega.Expect(mcsession.ExitCode()).Should(gomega.Equal(-1)) + ginkgo.By("Checking machineControllerManager process is running") + gomega.Expect(mcmsession.ExitCode()).Should(gomega.Equal(-1)) ginkgo.By("Checking nodes in target cluster are healthy") gomega.Eventually( c.TargetCluster.GetNumberOfReadyNodes, @@ -682,7 +682,7 @@ func (c *IntegrationTestFramework) ControllerTests() { c.ControlCluster.McmClient. MachineV1alpha1(). Machines(controlClusterNamespace). - Delete(ctx, "test-machine", metav1.DeleteOptions{})). + Delete(ctx, testMachineName, metav1.DeleteOptions{})). Should(gomega.BeNil(), "No Errors while deleting machine") ginkgo.By("Waiting until test-machine machine object is deleted") @@ -692,7 +692,7 @@ func (c *IntegrationTestFramework) ControllerTests() { c.pollingInterval). Should(gomega.BeTrue()) - ginkgo.By("Waiting until number of ready nodes is equal to number of initial nodes") + ginkgo.By("Waiting until number of ready nodes is equal to number of initial nodes") gomega.Eventually( c.TargetCluster.GetNumberOfNodes, c.timeout, @@ -754,7 +754,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDeployment, _ = c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) machineDeployment.Spec.Replicas = 1 _, updateErr := c.ControlCluster.McmClient. MachineV1alpha1(). @@ -771,7 +771,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDeployment, err = c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if len(machineDeployment.Status.Conditions) != 2 { return false @@ -811,7 +811,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDployment, _ := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) machineDployment.Spec.Replicas = 6 _, updateErr := c.ControlCluster.McmClient. MachineV1alpha1(). @@ -841,7 +841,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDployment, _ := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) machineDployment.Spec.Replicas = 2 _, updateErr := c.ControlCluster.McmClient. MachineV1alpha1(). @@ -868,7 +868,7 @@ func (c *IntegrationTestFramework) ControllerTests() { ginkgo.It("should freeze and unfreeze machineset temporarily", func() { gomega.Eventually(func() int { return c.machineSetFreezeEventCount(ctx, controlClusterNamespace) - }).Should(gomega.Equal(initialEventCount + 2)) + }, c.timeout, c.pollingInterval).Should(gomega.Equal(initialEventCount + 2)) }) }) ginkgo.Context("updation to v2 machine-class and replicas=4", func() { @@ -879,7 +879,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDployment, _ := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) machineDployment.Spec.Template.Spec.Class.Name = testMachineClassResources[1] machineDployment.Spec.Replicas = 4 _, updateErr := c.ControlCluster.McmClient. @@ -895,7 +895,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDeployment, err := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) if err != nil { log.Println("Failed to get machinedeployment object") } @@ -906,7 +906,7 @@ func (c *IntegrationTestFramework) ControllerTests() { machineDeployment, err := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) if err != nil { log.Println("Failed to get machinedeployment object") } @@ -932,7 +932,7 @@ func (c *IntegrationTestFramework) ControllerTests() { _, err := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Get(ctx, "test-machine-deployment", metav1.GetOptions{}) + Get(ctx, testMachineDeploymentName, metav1.GetOptions{}) if err == nil { ginkgo.By("Checking for errors") gomega.Expect( @@ -941,7 +941,7 @@ func (c *IntegrationTestFramework) ControllerTests() { MachineDeployments(controlClusterNamespace). Delete( ctx, - "test-machine-deployment", + testMachineDeploymentName, metav1.DeleteOptions{}, )). Should(gomega.BeNil()) @@ -986,49 +986,47 @@ func (c *IntegrationTestFramework) Cleanup() { ginkgo.By("Running Cleanup") //running locally, means none of the image is specified - if len(os.Getenv("MC_CONTAINER_IMAGE")) == 0 && len(os.Getenv("MCM_CONTAINER_IMAGE")) == 0 { - for i := 0; i < 5; i++ { - if mcsession.ExitCode() != -1 { - ginkgo.By("Restarting Machine Controller ") - outputFile, err := rotateOrAppendLogFile(mcLogFile, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = outputFile.WriteString("\n------------RESTARTED MC------------\n") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - args := strings.Fields( - fmt.Sprintf( - "make --directory=%s start CONTROL_KUBECONFIG=%s TARGET_KUBECONFIG=%s CONTROL_NAMESPACE=%s LEADER_ELECT=false ", - "../../..", - c.ControlCluster.KubeConfigFilePath, - c.TargetCluster.KubeConfigFilePath, - controlClusterNamespace), - ) - mcsession, err = gexec.Start(exec.Command(args[0], args[1:]...), outputFile, outputFile) // #nosec G204 -- Test only - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - break - } - time.Sleep(2 * time.Second) + for range 5 { + if mcsession.ExitCode() != -1 { + ginkgo.By("Restarting Machine Controller ") + outputFile, err := rotateOrAppendLogFile(mcLogFile, false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = outputFile.WriteString("\n------------RESTARTED MC------------\n") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + args := strings.Fields( + fmt.Sprintf( + "make --directory=%s start CONTROL_KUBECONFIG=%s TARGET_KUBECONFIG=%s CONTROL_NAMESPACE=%s LEADER_ELECT=false ", + "../../..", + c.ControlCluster.KubeConfigFilePath, + c.TargetCluster.KubeConfigFilePath, + controlClusterNamespace), + ) + mcsession, err = gexec.Start(exec.Command(args[0], args[1:]...), outputFile, outputFile) // #nosec G204 -- Test only + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + break } - for i := 0; i < 5; i++ { - if mcmsession.ExitCode() != -1 { - ginkgo.By("Restarting Machine Controller Manager") - outputFile, err := rotateOrAppendLogFile(mcmLogFile, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = outputFile.WriteString("\n------------RESTARTED MCM------------\n") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - args := strings.Fields( - fmt.Sprintf( - "make --directory=%s start CONTROL_KUBECONFIG=%s TARGET_KUBECONFIG=%s CONTROL_NAMESPACE=%s LEADER_ELECT=false MACHINE_SAFETY_OVERSHOOTING_PERIOD=300ms", - mcmRepoPath, - c.ControlCluster.KubeConfigFilePath, - c.TargetCluster.KubeConfigFilePath, - controlClusterNamespace), - ) - mcmsession, err = gexec.Start(exec.Command(args[0], args[1:]...), outputFile, outputFile) // #nosec G204 -- Test only - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - break - } - time.Sleep(2 * time.Second) + time.Sleep(2 * time.Second) + } + for range 5 { + if mcmsession.ExitCode() != -1 { + ginkgo.By("Restarting Machine Controller Manager") + outputFile, err := rotateOrAppendLogFile(mcmLogFile, false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = outputFile.WriteString("\n------------RESTARTED MCM------------\n") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + args := strings.Fields( + fmt.Sprintf( + "make --directory=%s start CONTROL_KUBECONFIG=%s TARGET_KUBECONFIG=%s CONTROL_NAMESPACE=%s LEADER_ELECT=false MACHINE_SAFETY_OVERSHOOTING_PERIOD=300ms", + mcmRepoPath, + c.ControlCluster.KubeConfigFilePath, + c.TargetCluster.KubeConfigFilePath, + controlClusterNamespace), + ) + mcmsession, err = gexec.Start(exec.Command(args[0], args[1:]...), outputFile, outputFile) // #nosec G204 -- Test only + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + break } + time.Sleep(2 * time.Second) } if preserveCRDuringCleanup == "true" { @@ -1037,10 +1035,13 @@ func (c *IntegrationTestFramework) Cleanup() { timeout := int64(900) c.cleanTestResources(ctx, timeout) } + + ginkgo.By("Killing any existing processes") + stopMCM(ctx) + if isControlSeed == "true" { - // scale back up the MCM deployment to 1 in the Control Cluster // This is needed when IT suite runs locally against a Control & Target cluster - + ginkgo.By("Scale back the existing machine controllers") if err := c.scaleMcmDeployment(1); err != nil { log.Println(err.Error()) } @@ -1055,17 +1056,15 @@ func (c *IntegrationTestFramework) Cleanup() { } } - ginkgo.By("Killing any existing processes") - stopMCM(ctx) } func (c *IntegrationTestFramework) cleanTestResources(ctx context.Context, timeout int64) { // Check and delete machinedeployment resource - if err := c.cleanMachineDeployment(ctx, "test-machine-deployment", timeout); err != nil { + if err := c.cleanMachineDeployment(ctx, testMachineDeploymentName, timeout); err != nil { log.Println(err.Error()) } // Check and delete machine resource - if err := c.cleanMachine(ctx, "test-machine", timeout); err != nil { + if err := c.cleanMachine(ctx, testMachineName, timeout); err != nil { log.Println(err.Error()) } @@ -1086,10 +1085,14 @@ func (c *IntegrationTestFramework) cleanMachineDeployment(ctx context.Context, n return err } log.Printf("deleting %s\n", name) + selector := fmt.Sprintf("metadata.name=%s", name) watchMachinesDepl, _ := c.ControlCluster.McmClient. MachineV1alpha1(). MachineDeployments(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineDeploymentObj.ResourceVersion + Watch(ctx, metav1.ListOptions{ + TimeoutSeconds: &timeout, + FieldSelector: selector, + }) for event := range watchMachinesDepl.ResultChan() { gomega.Expect(c.ControlCluster.McmClient. MachineV1alpha1(). @@ -1112,10 +1115,14 @@ func (c *IntegrationTestFramework) cleanMachine(ctx context.Context, name string return err } log.Printf("deleting %s\n", name) + selector := fmt.Sprintf("metadata.name=%s", name) watchMachines, _ := c.ControlCluster.McmClient. MachineV1alpha1(). Machines(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) //ResourceVersion: machineObj.ResourceVersion + Watch(ctx, metav1.ListOptions{ + TimeoutSeconds: &timeout, + FieldSelector: selector, + }) //ResourceVersion: machineObj.ResourceVersion for event := range watchMachines.ResultChan() { gomega.Expect(c.ControlCluster.McmClient. MachineV1alpha1(). @@ -1139,10 +1146,14 @@ func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machin } log.Printf("deleting %s machineclass", machineClassName) + selector := fmt.Sprintf("metadata.name=%s", machineClassName) watchMachineClass, _ := c.ControlCluster.McmClient. MachineV1alpha1(). MachineClasses(controlClusterNamespace). - Watch(ctx, metav1.ListOptions{TimeoutSeconds: &timeout}) + Watch(ctx, metav1.ListOptions{ + TimeoutSeconds: &timeout, + FieldSelector: selector, + }) for event := range watchMachineClass.ResultChan() { gomega.Expect(c.ControlCluster.McmClient. MachineV1alpha1(). @@ -1158,8 +1169,8 @@ func (c *IntegrationTestFramework) cleanMachineClass(ctx context.Context, machin func stopMCM(ctx context.Context) { processesToKill := []string{ - "machine-controller-manager --control-kubeconfig", // virt - "machine-controller --control-kubeconfig", // virt + "machine-controller-manager --control-kubeconfig", // virtual + "machine-controller --control-kubeconfig", // virtual "controller_manager --control-kubeconfig", "main --control-kubeconfig", "go run cmd/machine-controller/main.go", @@ -1238,7 +1249,7 @@ func (c *IntegrationTestFramework) getTestMachineSets(ctx context.Context, names testMachineSets := []string{} gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, machineSet := range machineSets.Items { - if machineSet.OwnerReferences[0].Name == "test-machine-deployment" { + if machineSet.OwnerReferences[0].Name == testMachineDeploymentName { testMachineSets = append(testMachineSets, machineSet.Name) } } @@ -1251,13 +1262,13 @@ func (c *IntegrationTestFramework) machineSetFreezeEventCount(ctx context.Contex for _, machineSet := range testMachineSets { for _, reason := range []string{"reason=FrozeMachineSet", "reason=UnfrozeMachineSet"} { event := fmt.Sprintf("%s,involvedObject.name=%s", reason, machineSet) - // TODO: handle error - frozenEvents, _ := c.ControlCluster.Clientset. + frozenEvents, err := c.ControlCluster.Clientset. CoreV1(). Events(namespace). List(ctx, metav1.ListOptions{ FieldSelector: event, }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) eventCount += len(frozenEvents.Items) } } From e34f396163463f39d1c2eeabbd184deed07d42b3 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 3 Mar 2025 10:55:53 +0530 Subject: [PATCH 07/10] nit: rename to a sensible name --- pkg/test/integration/common/framework.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index 36e8d0f68..c4ef00b49 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -103,7 +103,7 @@ var ( gnaSecretNameLabelValue = os.Getenv("GNA_SECRET_NAME") // Specifies whether the CRDs should be preserved during cleanup at the end - preserveCRDuringCleanup = os.Getenv("PRESERVE_CRD_AT_END") + preserveCRDDuringCleanup = os.Getenv("PRESERVE_CRD_AT_END") // Are the tests running for the virtual provider isVirtualProvider = os.Getenv("IS_VIRTUAL_PROVIDER") @@ -1029,8 +1029,8 @@ func (c *IntegrationTestFramework) Cleanup() { time.Sleep(2 * time.Second) } - if preserveCRDuringCleanup == "true" { - log.Printf("Preserve CRD: %s\n", preserveCRDuringCleanup) + if preserveCRDDuringCleanup == "true" { + log.Printf("Preserve CRD: %s\n", preserveCRDDuringCleanup) } else if c.ControlCluster.McmClient != nil { timeout := int64(900) c.cleanTestResources(ctx, timeout) From bf357eb5b936443bc60c53f75420a91726101003 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 10 Mar 2025 10:47:56 +0530 Subject: [PATCH 08/10] Added constants for events, address review comments --- pkg/controller/machine_safety.go | 9 +++++++-- pkg/test/integration/common/framework.go | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/controller/machine_safety.go b/pkg/controller/machine_safety.go index 1d5d8474a..0f17f367f 100644 --- a/pkg/controller/machine_safety.go +++ b/pkg/controller/machine_safety.go @@ -9,6 +9,7 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" intstrutil "k8s.io/apimachinery/pkg/util/intstr" @@ -26,6 +27,10 @@ const ( MachineDeploymentStateSync = "MachineDeploymentStateSync" // UnfreezeAnnotation indicates the controllers to unfreeze this object UnfreezeAnnotation = "safety.machine.sapcloud.io/unfreeze" + // MachineSetFreezeEvent is recorded when a machineset is frozen + MachineSetFreezeEvent = "FrozeMachineSet" + // MachineSetUnfreezeEvent is recorded when a machineset is unfrozen + MachineSetUnfreezeEvent = "UnfrozeMachineSet" ) // reconcileClusterMachineSafetyOvershooting checks all machineSet/machineDeployment @@ -330,7 +335,7 @@ func (c *controller) freezeMachineSetAndDeployment(ctx context.Context, machineS } } - c.recorder.Eventf(machineSet, "Normal", "FrozeMachineSet", "SafetyController: Froze MachineSet %s due to replica overshooting", machineSet.Name) + c.recorder.Eventf(machineSet, corev1.EventTypeNormal, MachineSetFreezeEvent, "SafetyController: Froze MachineSet %s due to replica overshooting", machineSet.Name) klog.V(2).Infof("SafetyController: Froze MachineSet %q due to overshooting of replicas", machineSet.Name) return nil } @@ -399,7 +404,7 @@ func (c *controller) unfreezeMachineSet(ctx context.Context, machineSet *v1alpha return err } - c.recorder.Eventf(machineSet, "Normal", "UnfrozeMachineSet", "SafetyController: Unfroze MachineSet %s", machineSet.Name) + c.recorder.Eventf(machineSet, corev1.EventTypeNormal, MachineSetUnfreezeEvent, "SafetyController: Unfroze MachineSet %s", machineSet.Name) klog.V(2).Infof("SafetyController: Unfroze MachineSet %q", machineSet.Name) return nil } diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index c4ef00b49..ed9f54364 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -565,6 +565,10 @@ func (c *IntegrationTestFramework) SetupBeforeSuite() { checkMcmRepoAvailable() + // When running the IT with virtual provider, there's no need to + // scale-down MC processes since none are running in the virtual cluster. + // Additionally, currently whenever running IT with the virtual provider, + // ControlCluster is assumed to be a seed since that simplifies the setup. if isControlSeed == "true" && isVirtualProvider != "true" { ginkgo.By("Scaledown existing machine controllers") gomega.Expect(c.scaleMcmDeployment(0)).To(gomega.BeNil()) @@ -1259,8 +1263,10 @@ func (c *IntegrationTestFramework) getTestMachineSets(ctx context.Context, names func (c *IntegrationTestFramework) machineSetFreezeEventCount(ctx context.Context, namespace string) int { eventCount := 0 testMachineSets := c.getTestMachineSets(ctx, namespace) + machineSetFreezeReason := fmt.Sprintf("reason=%s", controller.MachineSetFreezeEvent) + machineSetUnfreezeReason := fmt.Sprintf("reason=%s", controller.MachineSetUnfreezeEvent) for _, machineSet := range testMachineSets { - for _, reason := range []string{"reason=FrozeMachineSet", "reason=UnfrozeMachineSet"} { + for _, reason := range []string{machineSetFreezeReason, machineSetUnfreezeReason} { event := fmt.Sprintf("%s,involvedObject.name=%s", reason, machineSet) frozenEvents, err := c.ControlCluster.Clientset. CoreV1(). From 8024b71ae3cb2185f9349627d93f0b4eb8d37a36 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 10 Mar 2025 12:23:18 +0530 Subject: [PATCH 09/10] bugfix: add supported argument on fetching process to kill --- pkg/test/integration/common/framework.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/test/integration/common/framework.go b/pkg/test/integration/common/framework.go index ed9f54364..ae6bcb6d3 100644 --- a/pkg/test/integration/common/framework.go +++ b/pkg/test/integration/common/framework.go @@ -1213,7 +1213,7 @@ func findAndKillProcess(ctx context.Context, prefix string) (pids []int, err err } func findPidsByPrefix(ctx context.Context, prefix string) (pids []int, err error) { - cmd := exec.CommandContext(ctx, "ps", "-e", "-o", "pid,command") + cmd := exec.CommandContext(ctx, "ps", "-e", "-o", "pid,args") psOutput, err := cmd.Output() if err != nil { log.Printf("FindProcess could not run ps command: %v", err) From 29985390eb9ece91188ec15fc235c3db28277530 Mon Sep 17 00:00:00 2001 From: Prashant Tak Date: Mon, 10 Mar 2025 16:28:41 +0530 Subject: [PATCH 10/10] chore: update ginkgo version and flags Ref: https://onsi.github.io/ginkgo/MIGRATING_TO_V2#changed-command-line-flags Ref: https://onsi.github.io/ginkgo/MIGRATING_TO_V2#improved-profiling-support --- .ci/test | 4 ++-- go.mod | 22 +++++++++++----------- go.sum | 44 ++++++++++++++++++++++---------------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.ci/test b/.ci/test index d13330b9d..a496c4fec 100755 --- a/.ci/test +++ b/.ci/test @@ -36,7 +36,7 @@ function test_with_coverage() { local output_dir=test/output local coverprofile_file=coverprofile.out mkdir -p test/output - ginkgo $GINKGO_COMMON_FLAGS --coverprofile ${coverprofile_file} -covermode=set -outputdir ${output_dir} ${TEST_PACKAGES} + ginkgo $GINKGO_COMMON_FLAGS --coverprofile ${coverprofile_file} -covermode=set -output-dir ${output_dir} ${TEST_PACKAGES} sed -i -e '/mode: set/d' ${output_dir}/${coverprofile_file} {( echo "mode: set"; cat ${output_dir}/${coverprofile_file} )} > ${output_dir}/${coverprofile_file}.temp @@ -51,7 +51,7 @@ if [[ "${SKIP_UNIT_TESTS}" != "" ]]; then else echo ">>>>> Invoking unit tests" TEST_PACKAGES="cmd pkg" - GINKGO_COMMON_FLAGS="-r -timeout=1h0m0s --randomizeAllSpecs --randomizeSuites --failOnPending --progress" + GINKGO_COMMON_FLAGS="-r -timeout=1h0m0s --randomize-all --randomize-suites --fail-on-pending --show-node-events" test_with_coverage echo ">>>>> Finished executing unit tests" fi diff --git a/go.mod b/go.mod index a6969079b..e9a5ce81e 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/google/uuid v1.6.0 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 + github.com/onsi/ginkgo/v2 v2.23.0 + github.com/onsi/gomega v1.36.2 github.com/prometheus/client_golang v1.19.1 github.com/spf13/pflag v1.0.5 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 @@ -43,7 +43,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -58,16 +58,16 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/mod v0.23.0 // indirect + golang.org/x/net v0.35.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/tools v0.30.0 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 4ca168c08..4eee5a452 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -75,10 +75,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/ginkgo/v2 v2.23.0 h1:FA1xjp8ieYDzlgS5ABTpdUDB7wtngggONc8a7ku2NqQ= +github.com/onsi/ginkgo/v2 v2.23.0/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -126,32 +126,32 @@ golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -159,14 +159,14 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= +golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=