UPSTREAM: <carry>: [OTE] Add webhook tests skipping flake test

Skip "should be tolerant to openshift-service-ca certificate rotation". More info: https://issues.redhat.com/browse/OCPBUGS-60564
- Add dumping of container logs and `kubectl describe pods` output for better diagnostics.
- Include targeted certificate details dump (`tls.crt` parse) when failures occur.
- Add additional check to verify webhook responsiveness after certificate rotation.

This change is a refactor of code from openshift/origin#30059.

Assisted-by: Gemini

Author: camilamacedo86

openshift/tests-extension/.openshift-tests-extension/openshift_payload_olmv1.json

@@ -48,5 +48,55 @@
     "source": "openshift:payload:olmv1",
     "lifecycle": "blocking",
     "environmentSelector": {}
+  },
+  {
+    "name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working validating webhook",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:olmv1",
+    "lifecycle": "blocking",
+    "environmentSelector": {}
+  },
+  {
+    "name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working mutating webhook",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:olmv1",
+    "lifecycle": "blocking",
+    "environmentSelector": {}
+  },
+  {
+    "name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working conversion webhook",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:olmv1",
+    "lifecycle": "blocking",
+    "environmentSelector": {}
+  },
+  {
+    "name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should be tolerant to openshift-service-ca certificate rotation",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:olmv1",
+    "lifecycle": "blocking",
+    "environmentSelector": {}
+  },
+  {
+    "name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should be tolerant to tls secret deletion",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:olmv1",
+    "lifecycle": "blocking",
+    "environmentSelector": {}
   }
 ]

openshift/tests-extension/pkg/helpers/catalogs.go

@@ -0,0 +1,52 @@
+package helpers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	//nolint:staticcheck // ST1001: dot-imports for readability
+	. "github.com/onsi/gomega"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	olmv1 "github.com/operator-framework/operator-controller/api/v1"
+
+	"github/operator-framework-operator-controller/openshift/tests-extension/pkg/env"
+)
+
+// NewClusterCatalog returns a new ClusterCatalog object.
+// It sets the image reference as source.
+func NewClusterCatalog(name, imageRef string) *olmv1.ClusterCatalog {
+	return &olmv1.ClusterCatalog{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: olmv1.ClusterCatalogSpec{
+			Source: olmv1.CatalogSource{
+				Type: olmv1.SourceTypeImage,
+				Image: &olmv1.ImageSource{
+					Ref: imageRef,
+				},
+			},
+		},
+	}
+}
+
+// ExpectCatalogToBeServing checks that the catalog with the given name is installed
+func ExpectCatalogToBeServing(ctx context.Context, name string) {
+	k8sClient := env.Get().K8sClient
+	Eventually(func(g Gomega) {
+		var catalog olmv1.ClusterCatalog
+		err := k8sClient.Get(ctx, client.ObjectKey{Name: name}, &catalog)
+		g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get catalog %q", name))
+
+		conditions := catalog.Status.Conditions
+		g.Expect(conditions).NotTo(BeEmpty(), fmt.Sprintf("catalog %q has empty status.conditions", name))
+
+		g.Expect(meta.IsStatusConditionPresentAndEqual(conditions, olmv1.TypeServing, metav1.ConditionTrue)).
+			To(BeTrue(), fmt.Sprintf("catalog %q is not serving", name))
+	}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())
+}

openshift/tests-extension/pkg/helpers/commands.go

@@ -0,0 +1,69 @@
+package helpers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	//nolint:staticcheck // ST1001: dot-imports for readability
+	. "github.com/onsi/ginkgo/v2"
+)
+
+// findK8sTool returns "oc" if available, otherwise "kubectl".
+// If we are running locally we either prefer to use oc since some tests
+// require it, or fallback to kubectl if oc is not available.
+func findK8sTool() (string, error) {
+	tools := []string{"oc", "kubectl"}
+	for _, t := range tools {
+		// First check if the tool is available in the PATH.
+		if _, err := exec.LookPath(t); err != nil {
+			continue
+		}
+		// Verify that the tool is working by checking its version.
+		if err := exec.Command(t, "version", "--client").Run(); err == nil {
+			return t, nil
+		}
+	}
+	return "", fmt.Errorf("no Kubernetes CLI client found (tried %s)",
+		strings.Join(tools, ", "))
+}
+
+// RunK8sCommand runs a Kubernetes CLI command and returns ONLY stdout.
+// If the command fails, stderr is included in the returned error (not mixed with stdout).
+func RunK8sCommand(ctx context.Context, args ...string) ([]byte, error) {
+	tool, err := findK8sTool()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := exec.CommandContext(ctx, tool, args...)
+	out, err := cmd.Output()
+	if err != nil {
+		var ee *exec.ExitError
+		if errors.As(err, &ee) {
+			stderr := strings.TrimSpace(string(ee.Stderr))
+			if stderr != "" {
+				return nil, fmt.Errorf("%s %s failed: %w\nstderr:\n%s",
+					tool, strings.Join(args, " "), err, stderr)
+			}
+		}
+		return nil, fmt.Errorf("%s %s failed: %w",
+			tool, strings.Join(args, " "), err)
+	}
+	return out, nil
+}
+
+// RunAndPrint runs a `kubectl/oc` command via RunK8sCommand and writes both stdout and stderr
+// to the GinkgoWriter. It also prints the exact command being run.
+func RunAndPrint(ctx context.Context, args ...string) {
+	fmt.Fprintf(GinkgoWriter, "\n[diag] running: oc %s\n", strings.Join(args, " "))
+	out, err := RunK8sCommand(ctx, args...)
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "[diag] command failed: %v\n", err)
+	}
+	if len(out) > 0 {
+		fmt.Fprintf(GinkgoWriter, "%s\n", string(out))
+	}
+}

openshift/tests-extension/pkg/helpers/dump.go

@@ -0,0 +1,57 @@
+package helpers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	//nolint:staticcheck // ST1001: dot-imports for readability
+	. "github.com/onsi/ginkgo/v2"
+)
+
+// GetAllPodLogs prints logs for all containers in all pods in the given namespace.
+func GetAllPodLogs(ctx context.Context, namespace string) {
+	fmt.Fprintf(GinkgoWriter, "\n[pod-logs] namespace=%s\n", namespace)
+
+	By("Getting all pods in the namespace")
+	namesOut, err := RunK8sCommand(ctx, "get", "pods", "-n", namespace, "-o", "name")
+	if err != nil {
+		fmt.Fprintf(GinkgoWriter, "failed to list pods: %v\n%s\n", err, string(namesOut))
+		return
+	}
+	lines := strings.Split(strings.TrimSpace(string(namesOut)), "\n")
+	if len(lines) == 0 || (len(lines) == 1 && strings.TrimSpace(lines[0]) == "") {
+		fmt.Fprintln(GinkgoWriter, "no pods found")
+		return
+	}
+
+	By(fmt.Sprintf("[pod-logs] namespace=%s\n", namespace))
+	for _, res := range lines {
+		res = strings.TrimSpace(res)
+		if res == "" {
+			continue
+		}
+		fmt.Fprintf(GinkgoWriter, "\n--- logs: %s @ %s ---\n", res, time.Now().Format(time.RFC3339))
+		logsOut, err := RunK8sCommand(
+			ctx,
+			"logs",
+			"-n", namespace,
+			"--all-containers",
+			"--prefix",
+			"--timestamps",
+			res,
+		)
+		if err != nil {
+			fmt.Fprintf(GinkgoWriter, "error fetching logs: %v\n%s\n", err, string(logsOut))
+			continue
+		}
+		_, _ = GinkgoWriter.Write(logsOut) // ignore write error by design
+	}
+}
+
+// DescribePods prints the `kubectl describe pods` output for all pods in a given namespace.
+func DescribePods(ctx context.Context, namespace string) {
+	fmt.Fprintf(GinkgoWriter, "\n[diag] === describe pods in namespace %q ===\n", namespace)
+	RunAndPrint(ctx, "describe", "pods", "-n", namespace)
+}

openshift/tests-extension/test/olmv1-incompatible.go

@@ -60,10 +60,6 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
 		}
 		By(fmt.Sprintf("testing against OCP %s", testVersion))
 
-		By("finding a k8s client")
-		cmdLine, err := getK8sCommandLineClient()
-		Expect(err).To(Succeed())
-
 		By("creating a new Namespace")
 		nsCleanup := createNamespace(nsName)
 		DeferCleanup(nsCleanup)
@@ -85,7 +81,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
 		DeferCleanup(fileCleanup)
 		By(fmt.Sprintf("created operator tarball %q", fileOperator))
 
-		By(fmt.Sprintf("starting the operator build with %q via RAW URL", cmdLine))
+		By("starting the operator build via RAW URL")
 		opArgs := []string{
 			"create",
 			"--raw",
@@ -96,7 +92,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
 			"-f",
 			fileOperator,
 		}
-		buildOperator := startBuild(cmdLine, opArgs...)
+		buildOperator := startBuild(opArgs...)
 
 		By(fmt.Sprintf("waiting for the build %q to finish", buildOperator.Name))
 		waitForBuildToFinish(ctx, buildOperator.Name, nsName)
@@ -114,7 +110,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
 		DeferCleanup(fileCleanup)
 		By(fmt.Sprintf("created catalog tarball %q", fileCatalog))
 
-		By(fmt.Sprintf("starting the catalog build with %q via RAW URL", cmdLine))
+		By("starting the catalog build via RAW URL")
 		catalogArgs := []string{
 			"create",
 			"--raw",
@@ -125,7 +121,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
 			"-f",
 			fileCatalog,
 		}
-		buildCatalog := startBuild(cmdLine, catalogArgs...)
+		buildCatalog := startBuild(catalogArgs...)
 
 		By(fmt.Sprintf("waiting for the build %q to finish", buildCatalog.Name))
 		waitForBuildToFinish(ctx, buildCatalog.Name, nsName)
@@ -386,18 +382,10 @@ func waitForClusterOperatorUpgradable(ctx SpecContext, name string) {
 	}).WithTimeout(5 * time.Minute).WithPolling(1 * time.Second).Should(Succeed())
 }
 
-func getK8sCommandLineClient() (string, error) {
-	s, err := exec.LookPath("kubectl")
-	if err != nil {
-		s, err = exec.LookPath("oc")
-	}
-	return s, err
-}
-
-func startBuild(cmdLine string, args ...string) *buildv1.Build {
-	cmd := exec.Command(cmdLine, args...)
-	output, err := cmd.Output()
+func startBuild(args ...string) *buildv1.Build {
+	output, err := helpers.RunK8sCommand(context.Background(), args...)
 	Expect(err).To(Succeed(), printExitError(err))
+
 	/* The output is JSON of a build.build.openshift.io resource */
 	build := &buildv1.Build{}
 	Expect(json.Unmarshal(output, build)).To(Succeed(), "failed to unmarshal build")