Skip to content

Commit 88c9905

Browse files
wkingwking
committed
test/extended/cli/adm_upgrade/recommend: Trust the ingress CA
Avoid [1]: : [Serial][sig-cli] oc adm upgrade recommend When the update service has conditional recommendations runs successfully with an accepted conditional recommendation to the --version target [Suite:openshift/conformance/serial] 20s { fail [github.com/openshift/origin/test/extended/cli/adm_upgrade/recommend.go:225]: Unexpected error: <*errors.errorString | 0xc008463cc0>: expected: Failed to check for at least some preconditions: failed to get alerts from Thanos: unable to get /api/v1/alerts from URI in the openshift-monitoring/thanos-querier Route: thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX->Get "https://thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX/api/v1/alerts": tls: failed to verify certificate: x509: certificate signed by unknown authority by retrieving the default (self-signed) ingress certificate and passing it to 'oc' via --certificate-authority and a temporary file. The default ingress certificate knobs I'm using are documented in [2]. The router-certs-default fallback may not be documented, but is backed by [3,4]. We need to keep trusting the Kube API server though, and while I'm not clear on the mechanism it uses to pick its external API server certificate, iterating over all the TLS secrets in the Kube API server namespace and aggregating seems like it should pick up the one we need. [1]: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/30113/pull-ci-openshift-origin-main-e2e-aws-ovn-serial-2of2/1956090254670696448 [2]: https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/security_and_compliance/configuring-certificates#replacing-default-ingress_replacing-default-ingress [3]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/certificate/default_cert.go#L76-L83 [4]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/names.go#L152-L159 [5]: https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets
1 parent b18aac0 commit 88c9905

File tree

1 file changed

+79
-14
lines changed

1 file changed

+79
-14
lines changed

‎test/extended/cli/adm_upgrade/recommend.go‎

Lines changed: 79 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"net"
77
"net/url"
8+
"os"
89
"strconv"
910
"strings"
1011
"text/template"
@@ -31,6 +32,7 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
3132
oc := exutil.NewCLIWithFramework(f).AsAdmin()
3233
var cv *configv1.ClusterVersion
3334
var restoreChannel, restoreUpstream bool
35+
var caBundleFilePath string
3436

3537
g.BeforeAll(func() {
3638
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
@@ -51,6 +53,10 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
5153
if restoreUpstream {
5254
oc.Run("patch", "clusterversions.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/upstream", "value": "%s"}]`, cv.Spec.Upstream)).Execute()
5355
}
56+
57+
if caBundleFilePath != "" {
58+
os.Remove(caBundleFilePath)
59+
}
5460
})
5561

5662
g.It("runs successfully, even without upstream OpenShift Update Service customization", func() {
@@ -110,6 +116,7 @@ No updates available. You may still upgrade to a specific release image.*`)
110116

111117
g.Context("When the update service has conditional recommendations", func() {
112118
var currentVersion *semver.Version
119+
var token string
113120

114121
g.BeforeAll(func() {
115122
isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient())
@@ -175,15 +182,38 @@ No updates available. You may still upgrade to a specific release image.*`)
175182
}
176183
restoreUpstream = true
177184

185+
defaultIngressCert, err := getDefaultIngressCertificate(ctx, oc)
186+
o.Expect(err).NotTo(o.HaveOccurred())
187+
188+
kubeCerts, err := getKubernetesAPIServerCertificates(ctx, oc)
189+
o.Expect(err).NotTo(o.HaveOccurred())
190+
191+
caBundleFile, err := os.CreateTemp("", "ca-bundle")
192+
caBundleFilePath = caBundleFile.Name()
193+
_, err = caBundleFile.WriteString(fmt.Sprintf("%s\n%s", defaultIngressCert, kubeCerts))
194+
o.Expect(err).NotTo(o.HaveOccurred())
195+
196+
// alert retrieval requires a token-based kubeconfig to avoid:
197+
// Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
198+
o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
199+
o.Expect(oc.Run("create").Args("clusterrolebinding", "test", "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
200+
token, err = oc.Run("create").Args("token", "test").Output()
201+
o.Expect(err).NotTo(o.HaveOccurred())
202+
178203
time.Sleep(16 * time.Second) // Give the CVO time to retrieve recommendations and push to status
179204
})
180205

181206
g.It("runs successfully when listing all updates", func() {
182-
out, err := oc.Run("adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
183-
o.Expect(err).NotTo(o.HaveOccurred())
184-
err = matchRegexp(out, `Upstream update service: http://.*
185-
Channel: test-channel [(]available channels: other-channel, test-channel[)]
207+
oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
208+
o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
209+
o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
186210

211+
out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
212+
o.Expect(err).NotTo(o.HaveOccurred())
213+
err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
214+
215+
Upstream update service: http://.*
216+
Channel: test-channel [(]available channels: other-channel, test-channel[)]
187217
Updates to 4.[0-9]*:
188218
189219
Version: 4[.][0-9]*[.]0
@@ -195,23 +225,18 @@ Updates to 4[.][0-9]*:
195225
VERSION *ISSUES
196226
4[.][0-9]*[.]999 *no known issues relevant to this cluster
197227
4[.][0-9]*[.]998 *no known issues relevant to this cluster`)
198-
o.Expect(err).NotTo(o.HaveOccurred())
228+
o.Expect(err).NotTo(o.HaveOccurred())
229+
})
199230
})
200231

201232
g.It("runs successfully with an accepted conditional recommendation to the --version target", func() {
202-
// alert retrieval requires a token-based kubeconfig to avoid:
203-
// Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
204-
o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
205-
o.Expect(oc.Run("create").Args("clusterrolebinding", "test", "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
206-
token, err := oc.Run("create").Args("token", "test").Output()
207-
o.Expect(err).NotTo(o.HaveOccurred())
208-
209233
oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
210234
o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
211235
o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
212236

213-
out, err := oc.Run("adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
214-
o.Expect(err).To(o.HaveOccurred())
237+
out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
238+
239+
o.Expect(err).NotTo(o.HaveOccurred())
215240
err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
216241
217242
Upstream update service: http://.*
@@ -305,3 +330,43 @@ python3 -m http.server --bind ::
305330
Path: "graph",
306331
}, nil
307332
}
333+
334+
func getDefaultIngressCertificate(ctx context.Context, oc *exutil.CLI) (string, error) {
335+
defaultIngressSecretName, err := oc.Run("get").Args("--namespace=openshift-ingress-operator", "-o", "jsonpath={.spec.defaultCertificate.name}", "ingresscontroller.operator.openshift.io", "default").Output()
336+
if err != nil {
337+
return "", err
338+
}
339+
340+
if defaultIngressSecretName == "" {
341+
defaultIngressSecretName = "router-certs-default"
342+
}
343+
344+
ingressNamespace := "openshift-ingress"
345+
defaultIngressCert, err := oc.Run("extract").Args("--namespace", ingressNamespace, fmt.Sprintf("secret/%s", defaultIngressSecretName), "--keys=tls.crt", "--to=-").Output()
346+
if err != nil {
347+
return "", err
348+
}
349+
defaultIngressCert = fmt.Sprintf("%s\n", defaultIngressCert) // ensure a trailing newline, even if the earlier Output() stripped trailing newlines
350+
framework.Logf("default ingress certificate from the %s secret in the %s namespace: %q", defaultIngressSecretName, ingressNamespace, defaultIngressCert)
351+
return defaultIngressCert, nil
352+
}
353+
354+
func getKubernetesAPIServerCertificates(ctx context.Context, oc *exutil.CLI) (string, error) {
355+
kubeNamespace := "openshift-kube-apiserver"
356+
secrets, err := oc.AdminKubeClient().CoreV1().Secrets(kubeNamespace).List(ctx, metav1.ListOptions{})
357+
if err != nil {
358+
return "", err
359+
}
360+
361+
certs := make([]string, 0, len(secrets.Items))
362+
for _, secret := range secrets.Items {
363+
if secret.Type != corev1.SecretTypeTLS {
364+
continue
365+
}
366+
certs = append(certs, string(secret.Data["tls.crt"]))
367+
}
368+
369+
kubeCerts := strings.Join(certs, "")
370+
framework.Logf("default Kubernetes certificates from TLS secrets in the %s namespace: %q", kubeNamespace, kubeCerts)
371+
return kubeCerts, nil
372+
}

0 commit comments

Comments
 (0)