diff --git a/test/e2e/internal/network/cilium_agents.go b/test/e2e/internal/network/cilium_agents.go index 5699c32a5c..c9c583377a 100644 --- a/test/e2e/internal/network/cilium_agents.go +++ b/test/e2e/internal/network/cilium_agents.go @@ -19,12 +19,15 @@ package network import ( "context" "encoding/json" + "errors" "fmt" + "os" "strings" corev1 "k8s.io/api/core/v1" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" kc "github.com/deckhouse/virtualization/test/e2e/internal/kubectl" ) @@ -53,30 +56,33 @@ func CheckCiliumAgents(ctx context.Context, kubectl kc.Kubectl, vmName, vmNamesp } // Check each Cilium agent pod + var errs []error for _, pod := range pods { + nodeIP := nodeInternalIP if pod.Spec.NodeName == nodeName { - // For pods on the same node as the VM - found, err := searchIPFromCiliumIPCache(kubectl, pod, vmIP, innaddrAny) - if err != nil { - return err - } + nodeIP = innaddrAny + } - if !found { - return fmt.Errorf("failed: not found cilium agent %s for VM's node %s", pod.Name, nodeName) - } - } else { - // For pods on different nodes - found, err := searchIPFromCiliumIPCache(kubectl, pod, vmIP, nodeInternalIP) - if err != nil { - return err - } + ipCache, err := getCiliumIPCache(kubectl, pod) + if err != nil { + errs = append(errs, fmt.Errorf("failed to get Cilium Agent's IPCache `%s` on the node `%s`: %w", pod.Name, nodeName, err)) + continue + } - if !found { - return fmt.Errorf("failed: not found cilium agent %s for node %s", pod.Name, pod.Spec.NodeName) + err = validateIPInCiliumIPCache(vmIP, nodeIP, ipCache) + if err != nil { + errs = append(errs, err) + err = dumpIPCache(ipCache, nodeName, pod.Name) + if err != nil { + errs = append(errs, err) } } } + if len(errs) > 0 { + return fmt.Errorf("the Cilium agent check has failed: %w", errors.Join(errs...)) + } + return nil } @@ -140,22 +146,36 @@ func getCiliumAgentPods(kubectl kc.Kubectl) ([]corev1.Pod, error) { return podList.Items, nil } -func searchIPFromCiliumIPCache(kubectl kc.Kubectl, pod corev1.Pod, vmIP, nodeIP string) (bool, error) { +func getCiliumIPCache(kubectl kc.Kubectl, pod corev1.Pod) (string, error) { cmd := fmt.Sprintf("-n %s exec %s -c cilium-agent -- cilium map get cilium_ipcache", pod.Namespace, pod.Name) result := kubectl.RawCommand(cmd, kc.MediumTimeout) if result.Error() != nil { - return false, fmt.Errorf("failed to execute command: %w", result.Error()) + return "", fmt.Errorf("failed to execute command `%s`: %w", cmd, result.Error()) } - output := result.StdOut() - lines := strings.Split(output, "\n") - found := false - for _, line := range lines { + return result.StdOut(), nil +} + +func validateIPInCiliumIPCache(vmIP, nodeIP, ipCache string) error { + lines := strings.SplitSeq(ipCache, "\n") + for line := range lines { if strings.Contains(line, vmIP) && strings.Contains(line, nodeIP) { - found = true - break + return nil } } - return found, nil + return fmt.Errorf("VM's IP `%s` not found in the Cilium agent's ipcache; NodeIP: `%s`", vmIP, nodeIP) +} + +func dumpIPCache(ipCache, nodeName, podName string) error { + ft := framework.GetFormattedTestCaseFullText() + tmpDir := framework.GetTMPDir() + + resFileName := fmt.Sprintf("%s/e2e_failed__%s__%s__%s__cilium_ipcache.yaml", tmpDir, ft, nodeName, podName) + err := os.WriteFile(resFileName, []byte(ipCache), 0o644) + if err != nil { + return fmt.Errorf("saving Cilium Agent's IPCache to file '%s' failed: %w", resFileName, err) + } + + return nil } diff --git a/test/e2e/legacy/complex.go b/test/e2e/legacy/complex.go index e6290eb921..9bdb11b4c7 100644 --- a/test/e2e/legacy/complex.go +++ b/test/e2e/legacy/complex.go @@ -189,8 +189,8 @@ var _ = Describe("ComplexTest", Ordered, func() { Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr()) vms := strings.Split(res.StdOut(), " ") - // Skip this check until the issue with cilium-agents is fixed. - // CheckCiliumAgents(kubectl, ns, vms...) + // There is a known issue with the Cilium agent check. + CheckCiliumAgents(kubectl, ns, vms...) CheckExternalConnection(externalHost, httpStatusOk, ns, vms...) }) }) diff --git a/test/e2e/legacy/vm_vpc.go b/test/e2e/legacy/vm_vpc.go index 7e894341ec..b89a182ce4 100644 --- a/test/e2e/legacy/vm_vpc.go +++ b/test/e2e/legacy/vm_vpc.go @@ -136,8 +136,8 @@ var _ = Describe("VirtualMachineAdditionalNetworkInterfaces", Ordered, func() { vms := strings.Split(res.StdOut(), " ") Expect(vms).NotTo(BeEmpty()) - // Skip this check until the issue with cilium-agents is fixed. - // CheckCiliumAgents(kubectl, ns, vms...) + // There is a known issue with the Cilium agent check. + CheckCiliumAgents(kubectl, ns, vms...) CheckExternalConnection(externalHost, httpStatusOk, ns, vms...) }) diff --git a/test/e2e/vm/connectivity.go b/test/e2e/vm/connectivity.go index fb98140246..9eab111f63 100644 --- a/test/e2e/vm/connectivity.go +++ b/test/e2e/vm/connectivity.go @@ -68,13 +68,13 @@ var _ = Describe("VirtualMachineConnectivity", func() { t.CheckCloudInitCompleted(framework.LongTimeout) }) - // Skip this check until the issue with cilium-agents is fixed. - // By("Check Cilium agents are properly configured for the VMs", func() { - // err := network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), t.VMa.Name, f.Namespace().Name) - // Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", t.VMa.Name) - // err = network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), t.VMb.Name, f.Namespace().Name) - // Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", t.VMb.Name) - // }) + // There is a known issue with the Cilium agent check. + By("Check Cilium agents are properly configured for the VMs", func() { + err := network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), t.VMa.Name, f.Namespace().Name) + Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", t.VMa.Name) + err = network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), t.VMb.Name, f.Namespace().Name) + Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", t.VMb.Name) + }) By("Check VMs can reach external network", func() { network.CheckExternalConnectivity(f, t.VMa.Name, network.ExternalHost, network.HTTPStatusOk) diff --git a/test/e2e/vm/migration.go b/test/e2e/vm/migration.go index d4660ed772..a342255852 100644 --- a/test/e2e/vm/migration.go +++ b/test/e2e/vm/migration.go @@ -128,13 +128,13 @@ var _ = Describe("VirtualMachineMigration", func() { util.UntilVMMigrationSucceeded(crclient.ObjectKeyFromObject(vmUEFI), framework.LongTimeout) }) - // Skip this check until the issue with cilium-agents is fixed. - // By("Check Cilium agents are properly configured for the VM", func() { - // err := network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), vmBIOS.Name, f.Namespace().Name) - // Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", vmBIOS.Name) - // err = network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), vmUEFI.Name, f.Namespace().Name) - // Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", vmUEFI.Name) - // }) + // There is a known issue with the Cilium agent check. + By("Check Cilium agents are properly configured for the VM", func() { + err := network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), vmBIOS.Name, f.Namespace().Name) + Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", vmBIOS.Name) + err = network.CheckCiliumAgents(context.Background(), f.Clients.Kubectl(), vmUEFI.Name, f.Namespace().Name) + Expect(err).NotTo(HaveOccurred(), "Cilium agents check should succeed for VM %s", vmUEFI.Name) + }) By("Check VM can reach external network", func() { network.CheckExternalConnectivity(f, vmBIOS.Name, network.ExternalHost, network.HTTPStatusOk)