✨ Add PullSecret controller to save pull secret data locally (#425)

* ✨ Add PullSecret controller to save pull secret data locally

RFC: https://docs.google.com/document/d/1BXD6kj5zXHcGiqvJOikU2xs8kV26TPnzEKp6n7TKD4M/edit#heading=h.x3tfh25grvnv

* main.go: improved cache configuration for watching pull secret

Signed-off-by: Joe Lanford <joe.lanford@gmail.com>

---------

Signed-off-by: Joe Lanford <joe.lanford@gmail.com>

Author: anik120

cmd/manager/main.go

@@ -24,20 +24,28 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/containers/image/v5/types"
 	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	apimachineryrand "k8s.io/apimachinery/pkg/util/rand"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/metadata"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
+	crcache "sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -61,8 +69,8 @@ var (
 )
 
 const (
-	storageDir   = "catalogs"
-	authFilePath = "/etc/catalogd/auth.json"
+	storageDir     = "catalogs"
+	authFilePrefix = "catalogd-global-pull-secret"
 )
 
 func init() {
@@ -88,6 +96,7 @@ func main() {
 		keyFile              string
 		webhookPort          int
 		caCertDir            string
+		globalPullSecret     string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -105,6 +114,7 @@ func main() {
 	flag.StringVar(&keyFile, "tls-key", "", "The key file used for serving catalog contents over HTTPS. Requires tls-cert.")
 	flag.IntVar(&webhookPort, "webhook-server-port", 9443, "The port that the mutating webhook server serves at.")
 	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of CA certificate to use for verifying HTTPS connections to image registries.")
+	flag.StringVar(&globalPullSecret, "global-pull-secret", "", "The <namespace>/<name> of the global pull secret that is going to be used to pull bundle images.")
 
 	klog.InitFlags(flag.CommandLine)
 
@@ -120,6 +130,17 @@ func main() {
 
 	ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
 
+	authFilePath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%s.json", authFilePrefix, apimachineryrand.String(8)))
+	var globalPullSecretKey *k8stypes.NamespacedName
+	if globalPullSecret != "" {
+		secretParts := strings.Split(globalPullSecret, "/")
+		if len(secretParts) != 2 {
+			setupLog.Error(fmt.Errorf("incorrect number of components"), "value of global-pull-secret should be of the format <namespace>/<name>")
+			os.Exit(1)
+		}
+		globalPullSecretKey = &k8stypes.NamespacedName{Name: secretParts[1], Namespace: secretParts[0]}
+	}
+
 	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
 		setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together")
 		os.Exit(1)
@@ -148,6 +169,22 @@ func main() {
 		},
 	})
 
+	cacheOptions := crcache.Options{
+		ByObject: map[client.Object]crcache.ByObject{},
+	}
+	if globalPullSecretKey != nil {
+		cacheOptions.ByObject[&corev1.Secret{}] = crcache.ByObject{
+			Namespaces: map[string]crcache.Config{
+				globalPullSecretKey.Namespace: {
+					LabelSelector: k8slabels.Everything(),
+					FieldSelector: fields.SelectorFromSet(map[string]string{
+						"metadata.name": globalPullSecretKey.Name,
+					}),
+				},
+			},
+		}
+	}
+
 	// Create manager
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme,
@@ -159,6 +196,7 @@ func main() {
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "catalogd-operator-lock",
 		WebhookServer:          webhookServer,
+		Cache:                  cacheOptions,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create manager")
@@ -188,10 +226,20 @@ func main() {
 	}
 	unpacker := &source.ContainersImageRegistry{
 		BaseCachePath: unpackCacheBasePath,
-		SourceContext: &types.SystemContext{
-			OCICertPath:    caCertDir,
-			DockerCertPath: caCertDir,
-			AuthFilePath:   authFilePathIfPresent(setupLog),
+		SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) {
+			srcContext := &types.SystemContext{
+				DockerCertPath: caCertDir,
+				OCICertPath:    caCertDir,
+			}
+			if _, err := os.Stat(authFilePath); err == nil && globalPullSecretKey != nil {
+				logger.Info("using available authentication information for pulling image")
+				srcContext.AuthFilePath = authFilePath
+			} else if os.IsNotExist(err) {
+				logger.Info("no authentication information found for pulling image, proceeding without auth")
+			} else {
+				return nil, fmt.Errorf("could not stat auth file, error: %w", err)
+			}
+			return srcContext, nil
 		},
 	}
 
@@ -234,6 +282,19 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterCatalog")
 		os.Exit(1)
 	}
+
+	if globalPullSecretKey != nil {
+		setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", globalPullSecret)
+		err := (&corecontrollers.PullSecretReconciler{
+			Client:       mgr.GetClient(),
+			AuthFilePath: authFilePath,
+			SecretKey:    *globalPullSecretKey,
+		}).SetupWithManager(mgr)
+		if err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer")
+			os.Exit(1)
+		}
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -274,6 +335,10 @@ func main() {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
+	if err := os.Remove(authFilePath); err != nil {
+		setupLog.Error(err, "failed to cleanup temporary auth file")
+		os.Exit(1)
+	}
 }
 
 func podNamespace() string {
@@ -283,17 +348,3 @@ func podNamespace() string {
 	}
 	return string(namespace)
 }
-
-func authFilePathIfPresent(logger logr.Logger) string {
-	_, err := os.Stat(authFilePath)
-	if os.IsNotExist(err) {
-		logger.Info("auth file not found, skipping configuration of global auth file", "path", authFilePath)
-		return ""
-	}
-	if err != nil {
-		logger.Error(err, "unable to access auth file path", "path", authFilePath)
-		os.Exit(1)
-	}
-	logger.Info("auth file found, configuring globally for image registry interactions", "path", authFilePath)
-	return authFilePath
-}

internal/controllers/core/pull_secret_controller.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// PullSecretReconciler reconciles a specific Secret object
+// that contains global pull secrets for pulling Catalog images
+type PullSecretReconciler struct {
+	client.Client
+	SecretKey    types.NamespacedName
+	AuthFilePath string
+}
+
+func (r *PullSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	if req.Name != r.SecretKey.Name || req.Namespace != r.SecretKey.Namespace {
+		logger.Error(fmt.Errorf("received unexpected request for Secret %v/%v", req.Namespace, req.Name), "reconciliation error")
+		return ctrl.Result{}, nil
+	}
+
+	secret := &corev1.Secret{}
+	err := r.Get(ctx, req.NamespacedName, secret)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Info("secret not found")
+			return r.deleteSecretFile(logger)
+		}
+		logger.Error(err, "failed to get Secret")
+		return ctrl.Result{}, err
+	}
+
+	return r.writeSecretToFile(logger, secret)
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	_, err := ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}).
+		WithEventFilter(newSecretPredicate(r.SecretKey)).
+		Build(r)
+
+	return err
+}
+
+func newSecretPredicate(key types.NamespacedName) predicate.Predicate {
+	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
+		return obj.GetName() == key.Name && obj.GetNamespace() == key.Namespace
+	})
+}
+
+// writeSecretToFile writes the secret data to the specified file
+func (r *PullSecretReconciler) writeSecretToFile(logger logr.Logger, secret *corev1.Secret) (ctrl.Result, error) {
+	// image registry secrets are always stored with the key .dockerconfigjson
+	// ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials
+	dockerConfigJSON, ok := secret.Data[".dockerconfigjson"]
+	if !ok {
+		logger.Error(fmt.Errorf("expected secret.Data key not found"), "expected secret Data to contain key .dockerconfigjson")
+		return ctrl.Result{}, nil
+	}
+	// expected format for auth.json
+	// https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md
+	err := os.WriteFile(r.AuthFilePath, dockerConfigJSON, 0600)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to write secret data to file: %w", err)
+	}
+	logger.Info("saved global pull secret data locally")
+	return ctrl.Result{}, nil
+}
+
+// deleteSecretFile deletes the auth file if the secret is deleted
+func (r *PullSecretReconciler) deleteSecretFile(logger logr.Logger) (ctrl.Result, error) {
+	logger.Info("deleting local auth file", "file", r.AuthFilePath)
+	if err := os.Remove(r.AuthFilePath); err != nil {
+		if os.IsNotExist(err) {
+			logger.Info("auth file does not exist, nothing to delete")
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, fmt.Errorf("failed to delete secret file: %w", err)
+	}
+	logger.Info("auth file deleted successfully")
+	return ctrl.Result{}, nil
+}

internal/controllers/core/pull_secret_controller_test.go

@@ -0,0 +1,95 @@
+package core
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestSecretSyncerReconciler(t *testing.T) {
+	secretData := []byte(`{"auths":{"exampleRegistry": "exampledata"}}`)
+	authFileName := "test-auth.json"
+	for _, tt := range []struct {
+		name                  string
+		secret                *corev1.Secret
+		addSecret             bool
+		wantErr               string
+		fileShouldExistBefore bool
+		fileShouldExistAfter  bool
+	}{
+		{
+			name: "secret exists, content gets saved to authFile",
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-secret",
+					Namespace: "test-secret-namespace",
+				},
+				Data: map[string][]byte{
+					".dockerconfigjson": secretData,
+				},
+			},
+			addSecret:             true,
+			fileShouldExistBefore: false,
+			fileShouldExistAfter:  true,
+		},
+		{
+			name: "secret does not exist, file exists previously, file should get deleted",
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-secret",
+					Namespace: "test-secret-namespace",
+				},
+				Data: map[string][]byte{
+					".dockerconfigjson": secretData,
+				},
+			},
+			addSecret:             false,
+			fileShouldExistBefore: true,
+			fileShouldExistAfter:  false,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			tempAuthFile := filepath.Join(t.TempDir(), authFileName)
+			clientBuilder := fake.NewClientBuilder()
+			if tt.addSecret {
+				clientBuilder = clientBuilder.WithObjects(tt.secret)
+			}
+			cl := clientBuilder.Build()
+
+			secretKey := types.NamespacedName{Namespace: tt.secret.Namespace, Name: tt.secret.Name}
+			r := &PullSecretReconciler{
+				Client:       cl,
+				SecretKey:    secretKey,
+				AuthFilePath: tempAuthFile,
+			}
+			if tt.fileShouldExistBefore {
+				err := os.WriteFile(tempAuthFile, secretData, 0600)
+				require.NoError(t, err)
+			}
+			res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: secretKey})
+			if tt.wantErr == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, tt.wantErr)
+			}
+			require.Equal(t, ctrl.Result{}, res)
+
+			if tt.fileShouldExistAfter {
+				_, err := os.Stat(tempAuthFile)
+				require.NoError(t, err)
+			} else {
+				_, err := os.Stat(tempAuthFile)
+				require.True(t, os.IsNotExist(err))
+			}
+		})
+	}
+}