test/extended/cli/adm_upgrade/recommend: Trust the ingress CA

Avoid [1]:

  : [Serial][sig-cli] oc adm upgrade recommend When the update service has conditional recommendations runs successfully with an accepted conditional recommendation to the --version target [Suite:openshift/conformance/serial]	20s
  {  fail [github.com/openshift/origin/test/extended/cli/adm_upgrade/recommend.go:225]: Unexpected error:
    <*errors.errorString | 0xc008463cc0>:
    expected:
      Failed to check for at least some preconditions: failed to get alerts from Thanos: unable to get /api/v1/alerts from URI in the openshift-monitoring/thanos-querier Route: thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX->Get "https://thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX/api/v1/alerts": tls: failed to verify certificate: x509: certificate signed by unknown authority

by retrieving the default (self-signed) ingress certificate and
passing it to 'oc' via --certificate-authority and a temporary file.
The default ingress certificate knobs I'm using are documented in [2].
The router-certs-default fallback may not be documented, but is backed
by [3,4].

We need to keep trusting the Kube API server though, and while I'm not
clear on the mechanism it uses to pick its external API server
certificate, iterating over all the TLS secrets in the Kube API server
namespace and aggregating seems like it should pick up the one we
need.

[1]: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/30113/pull-ci-openshift-origin-main-e2e-aws-ovn-serial-2of2/1956090254670696448
[2]: https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/security_and_compliance/configuring-certificates#replacing-default-ingress_replacing-default-ingress
[3]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/certificate/default_cert.go#L76-L83
[4]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/names.go#L152-L159
[5]: https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets

Author: wking

test/extended/cli/adm_upgrade/recommend.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"text/template"
@@ -31,6 +32,7 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
 	oc := exutil.NewCLIWithFramework(f).AsAdmin()
 	var cv *configv1.ClusterVersion
 	var restoreChannel, restoreUpstream bool
+	var caBundleFilePath string
 
 	g.BeforeAll(func() {
 		isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
@@ -51,6 +53,10 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
 		if restoreUpstream {
 			oc.Run("patch", "clusterversions.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/upstream", "value": "%s"}]`, cv.Spec.Upstream)).Execute()
 		}
+
+		if caBundleFilePath != "" {
+			os.Remove(caBundleFilePath)
+		}
 	})
 
 	g.It("runs successfully, even without upstream OpenShift Update Service customization", func() {
@@ -110,6 +116,7 @@ No updates available. You may still upgrade to a specific release image.*`)
 
 	g.Context("When the update service has conditional recommendations", func() {
 		var currentVersion *semver.Version
+		var token string
 
 		g.BeforeAll(func() {
 			isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient())
@@ -175,15 +182,38 @@ No updates available. You may still upgrade to a specific release image.*`)
 			}
 			restoreUpstream = true
 
+			defaultIngressCert, err := getDefaultIngressCertificate(ctx, oc)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			kubeCerts, err := getKubernetesAPIServerCertificates(ctx, oc)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			caBundleFile, err := os.CreateTemp("", "ca-bundle")
+			caBundleFilePath = caBundleFile.Name()
+			_, err = caBundleFile.WriteString(fmt.Sprintf("%s\n%s", defaultIngressCert, kubeCerts))
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			// alert retrieval requires a token-based kubeconfig to avoid:
+			//   Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
+			o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
+			o.Expect(oc.Run("create").Args("clusterrolebinding", "test", "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
+			token, err = oc.Run("create").Args("token", "test").Output()
+			o.Expect(err).NotTo(o.HaveOccurred())
+
 			time.Sleep(16 * time.Second) // Give the CVO time to retrieve recommendations and push to status
 		})
 
 		g.It("runs successfully when listing all updates", func() {
-			out, err := oc.Run("adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
-			o.Expect(err).NotTo(o.HaveOccurred())
-			err = matchRegexp(out, `Upstream update service: http://.*
-Channel: test-channel [(]available channels: other-channel, test-channel[)]
+			oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
+				o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
+				o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
 
+				out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
+				o.Expect(err).NotTo(o.HaveOccurred())
+				err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
+
+Upstream update service: http://.*
+Channel: test-channel [(]available channels: other-channel, test-channel[)]
 Updates to 4.[0-9]*:
 
   Version: 4[.][0-9]*[.]0
@@ -195,22 +225,16 @@ Updates to 4[.][0-9]*:
   VERSION  *ISSUES
   4[.][0-9]*[.]999  *no known issues relevant to this cluster
   4[.][0-9]*[.]998  *no known issues relevant to this cluster`)
-			o.Expect(err).NotTo(o.HaveOccurred())
+				o.Expect(err).NotTo(o.HaveOccurred())
+			})
 		})
 
 		g.It("runs successfully with an accepted conditional recommendation to the --version target", func() {
-			// alert retrieval requires a token-based kubeconfig to avoid:
-			//   Failed to check for at least some preconditions: failed to get alerts from Thanos: no token is currently in use for this session
-			o.Expect(oc.Run("create").Args("serviceaccount", "test").Execute()).To(o.Succeed())
-			o.Expect(oc.Run("create").Args("clusterrolebinding", "test", "--clusterrole=cluster-admin", fmt.Sprintf("--serviceaccount=%s:test", oc.Namespace())).Execute()).To(o.Succeed())
-			token, err := oc.Run("create").Args("token", "test").Output()
-			o.Expect(err).NotTo(o.HaveOccurred())
-
 			oc.WithKubeConfigCopy(func(oc *exutil.CLI) {
 				o.Expect(oc.Run("config", "set-credentials").Args("test", "--token", token).Execute()).To(o.Succeed())
 				o.Expect(oc.Run("config", "set-context").Args("--current", "--user", "test").Execute()).To(o.Succeed())
 
-				out, err := oc.Run("adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
+				out, err := oc.Run("--certificate-authority", caBundleFilePath, "adm", "upgrade", "recommend", "--version", fmt.Sprintf("4.%d.0", currentVersion.Minor+1), "--accept", "ConditionalUpdateRisk").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_PRECHECK", "true").EnvVar("OC_ENABLE_CMD_UPGRADE_RECOMMEND_ACCEPT", "true").Output()
 				o.Expect(err).To(o.HaveOccurred())
 				err = matchRegexp(out, `The following conditions found no cause for concern in updating this cluster to later releases.*
 
@@ -305,3 +329,43 @@ python3 -m http.server --bind ::
 		Path:   "graph",
 	}, nil
 }
+
+func getDefaultIngressCertificate(ctx context.Context, oc *exutil.CLI) (string, error) {
+	defaultIngressSecretName, err := oc.Run("get").Args("--namespace=openshift-ingress-operator", "-o", "jsonpath={.spec.defaultCertificate.name}", "ingresscontroller.operator.openshift.io", "default").Output()
+	if err != nil {
+		return "", err
+	}
+
+	if defaultIngressSecretName == "" {
+		defaultIngressSecretName = "router-certs-default"
+	}
+
+	ingressNamespace := "openshift-ingress"
+	defaultIngressCert, err := oc.Run("extract").Args("--namespace", ingressNamespace, fmt.Sprintf("secret/%s", defaultIngressSecretName), "--keys=tls.crt", "--to=-").Output()
+	if err != nil {
+		return "", err
+	}
+	defaultIngressCert = fmt.Sprintf("%s\n", defaultIngressCert) // ensure a trailing newline, even if the earlier Output() stripped trailing newlines
+	framework.Logf("default ingress certificate from the %s secret in the %s namespace: %q", defaultIngressSecretName, ingressNamespace, defaultIngressCert)
+	return defaultIngressCert, nil
+}
+
+func getKubernetesAPIServerCertificates(ctx context.Context, oc *exutil.CLI) (string, error) {
+	kubeNamespace := "openshift-kube-apiserver"
+	secrets, err := oc.AdminKubeClient().CoreV1().Secrets(kubeNamespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	certs := make([]string, 0, len(secrets.Items))
+	for _, secret := range secrets.Items {
+		if secret.Type != corev1.SecretTypeTLS {
+			continue
+		}
+		certs = append(certs, string(secret.Data["tls.crt"]))
+	}
+
+	kubeCerts := strings.Join(certs, "")
+	framework.Logf("default Kubernetes certificates from TLS secrets in the %s namespace: %q", kubeNamespace, kubeCerts)
+	return kubeCerts, nil
+}