Skip to content

Commit 1f2debf

Browse files
camilamacedo86camilamacedo86
committed
UPSTREAM: <carry>: [OTE] Add webhook tests
- Add dumping of container logs and `kubectl describe pods` output for better diagnostics. - Include targeted certificate details dump (`tls.crt` parse) when failures occur. - Add additional check to verify webhook responsiveness after certificate rotation. This change is a refactor of code from openshift/origin#30059. Assisted-by: Gemini
1 parent 4cae159 commit 1f2debf

File tree

6 files changed

+679
-19
lines changed

6 files changed

+679
-19
lines changed

‎openshift/tests-extension/.openshift-tests-extension/openshift_payload_olmv1.json

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,5 +48,45 @@
4848
"source": "openshift:payload:olmv1",
4949
"lifecycle": "blocking",
5050
"environmentSelector": {}
51+
},
52+
{
53+
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working validating webhook",
54+
"labels": {},
55+
"resources": {
56+
"isolation": {}
57+
},
58+
"source": "openshift:payload:olmv1",
59+
"lifecycle": "blocking",
60+
"environmentSelector": {}
61+
},
62+
{
63+
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working mutating webhook",
64+
"labels": {},
65+
"resources": {
66+
"isolation": {}
67+
},
68+
"source": "openshift:payload:olmv1",
69+
"lifecycle": "blocking",
70+
"environmentSelector": {}
71+
},
72+
{
73+
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should have a working conversion webhook",
74+
"labels": {},
75+
"resources": {
76+
"isolation": {}
77+
},
78+
"source": "openshift:payload:olmv1",
79+
"lifecycle": "blocking",
80+
"environmentSelector": {}
81+
},
82+
{
83+
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should be tolerant to tls secret deletion",
84+
"labels": {},
85+
"resources": {
86+
"isolation": {}
87+
},
88+
"source": "openshift:payload:olmv1",
89+
"lifecycle": "blocking",
90+
"environmentSelector": {}
5191
}
5292
]

‎openshift/tests-extension/pkg/helpers/catalogs.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package helpers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
//nolint:staticcheck // ST1001: dot-imports for readability
9+
. "github.com/onsi/gomega"
10+
11+
"k8s.io/apimachinery/pkg/api/meta"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
15+
olmv1 "github.com/operator-framework/operator-controller/api/v1"
16+
17+
"github/operator-framework-operator-controller/openshift/tests-extension/pkg/env"
18+
)
19+
20+
// NewClusterCatalog returns a new ClusterCatalog object.
21+
// It sets the image reference as source.
22+
func NewClusterCatalog(name, imageRef string) *olmv1.ClusterCatalog {
23+
return &olmv1.ClusterCatalog{
24+
ObjectMeta: metav1.ObjectMeta{
25+
Name: name,
26+
},
27+
Spec: olmv1.ClusterCatalogSpec{
28+
Source: olmv1.CatalogSource{
29+
Type: olmv1.SourceTypeImage,
30+
Image: &olmv1.ImageSource{
31+
Ref: imageRef,
32+
},
33+
},
34+
},
35+
}
36+
}
37+
38+
// ExpectCatalogToBeServing checks that the catalog with the given name is installed
39+
func ExpectCatalogToBeServing(ctx context.Context, name string) {
40+
k8sClient := env.Get().K8sClient
41+
Eventually(func(g Gomega) {
42+
var catalog olmv1.ClusterCatalog
43+
err := k8sClient.Get(ctx, client.ObjectKey{Name: name}, &catalog)
44+
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get catalog %q", name))
45+
46+
conditions := catalog.Status.Conditions
47+
g.Expect(conditions).NotTo(BeEmpty(), fmt.Sprintf("catalog %q has empty status.conditions", name))
48+
49+
g.Expect(meta.IsStatusConditionPresentAndEqual(conditions, olmv1.TypeServing, metav1.ConditionTrue)).
50+
To(BeTrue(), fmt.Sprintf("catalog %q is not serving", name))
51+
}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())
52+
}

‎openshift/tests-extension/pkg/helpers/commands.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package helpers
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"os/exec"
8+
"strings"
9+
10+
//nolint:staticcheck // ST1001: dot-imports for readability
11+
. "github.com/onsi/ginkgo/v2"
12+
)
13+
14+
// findK8sTool returns "oc" if available, otherwise "kubectl".
15+
// If we are running locally we either prefer to use oc since some tests
16+
// require it, or fallback to kubectl if oc is not available.
17+
func findK8sTool() (string, error) {
18+
tools := []string{"oc", "kubectl"}
19+
for _, t := range tools {
20+
// First check if the tool is available in the PATH.
21+
if _, err := exec.LookPath(t); err != nil {
22+
continue
23+
}
24+
// Verify that the tool is working by checking its version.
25+
if err := exec.Command(t, "version", "--client").Run(); err == nil {
26+
return t, nil
27+
}
28+
}
29+
return "", fmt.Errorf("no Kubernetes CLI client found (tried %s)",
30+
strings.Join(tools, ", "))
31+
}
32+
33+
// RunK8sCommand runs a Kubernetes CLI command and returns ONLY stdout.
34+
// If the command fails, stderr is included in the returned error (not mixed with stdout).
35+
func RunK8sCommand(ctx context.Context, args ...string) ([]byte, error) {
36+
tool, err := findK8sTool()
37+
if err != nil {
38+
return nil, err
39+
}
40+
41+
cmd := exec.CommandContext(ctx, tool, args...)
42+
out, err := cmd.Output()
43+
if err != nil {
44+
var ee *exec.ExitError
45+
if errors.As(err, &ee) {
46+
stderr := strings.TrimSpace(string(ee.Stderr))
47+
if stderr != "" {
48+
return nil, fmt.Errorf("%s %s failed: %w\nstderr:\n%s",
49+
tool, strings.Join(args, " "), err, stderr)
50+
}
51+
}
52+
return nil, fmt.Errorf("%s %s failed: %w",
53+
tool, strings.Join(args, " "), err)
54+
}
55+
return out, nil
56+
}
57+
58+
// RunAndPrint runs a `kubectl/oc` command via RunK8sCommand and writes both stdout and stderr
59+
// to the GinkgoWriter. It also prints the exact command being run.
60+
func RunAndPrint(ctx context.Context, args ...string) {
61+
fmt.Fprintf(GinkgoWriter, "\n[diag] running: %s\n",
62+
strings.Join(quoteArgs(args), " "))
63+
out, err := RunK8sCommand(ctx, args...)
64+
if err != nil {
65+
fmt.Fprintf(GinkgoWriter, "[diag] command failed: %v\n", err)
66+
}
67+
if len(out) > 0 {
68+
fmt.Fprintf(GinkgoWriter, "%s\n", string(out))
69+
}
70+
}
71+
72+
func quoteArgs(args []string) []string {
73+
quoted := make([]string, len(args))
74+
for i, a := range args {
75+
// Add quotes only if whitespace or special chars are present
76+
if strings.ContainsAny(a, " \t") {
77+
quoted[i] = fmt.Sprintf("%q", a)
78+
} else {
79+
quoted[i] = a
80+
}
81+
}
82+
return quoted
83+
}

‎openshift/tests-extension/pkg/helpers/dump.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package helpers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
//nolint:staticcheck // ST1001: dot-imports for readability
9+
. "github.com/onsi/ginkgo/v2"
10+
)
11+
12+
func sectionHeader(format string, a ...any) {
13+
fmt.Fprintf(GinkgoWriter, "\n=== %s ===\n", fmt.Sprintf(format, a...))
14+
}
15+
16+
func subHeader(format string, a ...any) {
17+
fmt.Fprintf(GinkgoWriter, "\n--- %s ---\n", fmt.Sprintf(format, a...))
18+
}
19+
20+
// GetAllPodLogs prints logs for all containers in all pods in the given namespace.
21+
func GetAllPodLogs(ctx context.Context, namespace string) {
22+
sectionHeader("[pod-logs] namespace=%s", namespace)
23+
24+
By("listing pods in namespace " + namespace)
25+
namesOut, err := RunK8sCommand(ctx, "get", "pods", "-n", namespace, "-o", "name")
26+
if err != nil {
27+
fmt.Fprintf(GinkgoWriter, "failed to list pods: %v\n%s\n", err, string(namesOut))
28+
return
29+
}
30+
lines := strings.Fields(strings.TrimSpace(string(namesOut)))
31+
if len(lines) == 0 {
32+
fmt.Fprintln(GinkgoWriter, "(no pods found)")
33+
return
34+
}
35+
36+
for _, res := range lines {
37+
subHeader("logs for %s", res)
38+
logsOut, err := RunK8sCommand(
39+
ctx,
40+
"logs",
41+
"-n", namespace,
42+
"--all-containers",
43+
"--prefix",
44+
"--timestamps",
45+
res,
46+
)
47+
if err != nil {
48+
fmt.Fprintf(GinkgoWriter, "error fetching logs for %s: %v\n%s\n", res, err, string(logsOut))
49+
continue
50+
}
51+
_, _ = GinkgoWriter.Write(logsOut) // ignore write error by design
52+
}
53+
fmt.Fprintln(GinkgoWriter)
54+
}
55+
56+
// DescribePods prints the `kubectl/oc describe pods` output for all pods in a given namespace.
57+
func DescribePods(ctx context.Context, namespace string) {
58+
sectionHeader("[describe pods] namespace=%s", namespace)
59+
RunAndPrint(ctx, "describe", "pods", "-n", namespace)
60+
}
61+
62+
// DescribeAllClusterCatalogs lists all ClusterCatalogs and runs `describe` on each.
63+
func DescribeAllClusterCatalogs(ctx context.Context) {
64+
sectionHeader("[cluster catalogs]")
65+
66+
out, err := RunK8sCommand(ctx, "get", "clustercatalogs", "-o", "name")
67+
if err != nil {
68+
fmt.Fprintf(GinkgoWriter, "failed to list clustercatalogs: %v\n", err)
69+
return
70+
}
71+
72+
catalogs := strings.Fields(strings.TrimSpace(string(out)))
73+
if len(catalogs) == 0 {
74+
fmt.Fprintln(GinkgoWriter, "(no clustercatalogs found)")
75+
RunAndPrint(ctx, "get", "clustercatalogs")
76+
return
77+
}
78+
79+
for _, catalog := range catalogs {
80+
subHeader("describe %s", catalog)
81+
RunAndPrint(ctx, "describe", catalog)
82+
}
83+
fmt.Fprintln(GinkgoWriter)
84+
}
85+
86+
// DescribeAllClusterExtensions describes every ClusterExtension in the given namespace.
87+
func DescribeAllClusterExtensions(ctx context.Context, namespace string) {
88+
if namespace == "" {
89+
return
90+
}
91+
sectionHeader("[clusterextensions] namespace=%s", namespace)
92+
93+
args := []string{"get", "clusterextensions", "-n", namespace, "-o", "name"}
94+
out, err := RunK8sCommand(ctx, args...)
95+
if err != nil {
96+
fmt.Fprintf(GinkgoWriter, "failed to list clusterextensions: %v\n", err)
97+
RunAndPrint(ctx, args...)
98+
return
99+
}
100+
101+
names := strings.Fields(strings.TrimSpace(string(out)))
102+
if len(names) == 0 {
103+
fmt.Fprintln(GinkgoWriter, "(no clusterextensions found)")
104+
RunAndPrint(ctx, args...)
105+
return
106+
}
107+
108+
for _, n := range names {
109+
subHeader("describe %s", n)
110+
RunAndPrint(ctx, "describe", "clusterextension", strings.TrimPrefix(n, "clusterextension/"), "-n", namespace)
111+
}
112+
fmt.Fprintln(GinkgoWriter)
113+
}

‎openshift/tests-extension/test/olmv1-incompatible.go

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,6 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
6060
}
6161
By(fmt.Sprintf("testing against OCP %s", testVersion))
6262

63-
By("finding a k8s client")
64-
cmdLine, err := getK8sCommandLineClient()
65-
Expect(err).To(Succeed())
66-
6763
By("creating a new Namespace")
6864
nsCleanup := createNamespace(nsName)
6965
DeferCleanup(nsCleanup)
@@ -85,7 +81,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
8581
DeferCleanup(fileCleanup)
8682
By(fmt.Sprintf("created operator tarball %q", fileOperator))
8783

88-
By(fmt.Sprintf("starting the operator build with %q via RAW URL", cmdLine))
84+
By("starting the operator build via RAW URL")
8985
opArgs := []string{
9086
"create",
9187
"--raw",
@@ -96,7 +92,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
9692
"-f",
9793
fileOperator,
9894
}
99-
buildOperator := startBuild(cmdLine, opArgs...)
95+
buildOperator := startBuild(opArgs...)
10096

10197
By(fmt.Sprintf("waiting for the build %q to finish", buildOperator.Name))
10298
waitForBuildToFinish(ctx, buildOperator.Name, nsName)
@@ -114,7 +110,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
114110
DeferCleanup(fileCleanup)
115111
By(fmt.Sprintf("created catalog tarball %q", fileCatalog))
116112

117-
By(fmt.Sprintf("starting the catalog build with %q via RAW URL", cmdLine))
113+
By("starting the catalog build via RAW URL")
118114
catalogArgs := []string{
119115
"create",
120116
"--raw",
@@ -125,7 +121,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLM][Skipped:Disconnected] OLMv1
125121
"-f",
126122
fileCatalog,
127123
}
128-
buildCatalog := startBuild(cmdLine, catalogArgs...)
124+
buildCatalog := startBuild(catalogArgs...)
129125

130126
By(fmt.Sprintf("waiting for the build %q to finish", buildCatalog.Name))
131127
waitForBuildToFinish(ctx, buildCatalog.Name, nsName)
@@ -386,18 +382,10 @@ func waitForClusterOperatorUpgradable(ctx SpecContext, name string) {
386382
}).WithTimeout(5 * time.Minute).WithPolling(1 * time.Second).Should(Succeed())
387383
}
388384

389-
func getK8sCommandLineClient() (string, error) {
390-
s, err := exec.LookPath("kubectl")
391-
if err != nil {
392-
s, err = exec.LookPath("oc")
393-
}
394-
return s, err
395-
}
396-
397-
func startBuild(cmdLine string, args ...string) *buildv1.Build {
398-
cmd := exec.Command(cmdLine, args...)
399-
output, err := cmd.Output()
385+
func startBuild(args ...string) *buildv1.Build {
386+
output, err := helpers.RunK8sCommand(context.Background(), args...)
400387
Expect(err).To(Succeed(), printExitError(err))
388+
401389
/* The output is JSON of a build.build.openshift.io resource */
402390
build := &buildv1.Build{}
403391
Expect(json.Unmarshal(output, build)).To(Succeed(), "failed to unmarshal build")

0 commit comments

Comments
 (0)