diff --git a/Makefile b/Makefile index 13813a8d..5fa5c8b6 100644 --- a/Makefile +++ b/Makefile @@ -404,12 +404,12 @@ kind-up: ## Create a kind cluster for local development } @if $(KIND) get clusters | grep -q "^$(KIND_CLUSTER)$$"; then \ echo "Kind cluster '$(KIND_CLUSTER)' already exists."; \ + echo "==> Exporting kubeconfig to $(KIND_KUBECONFIG)"; \ + $(KIND) get kubeconfig --name $(KIND_CLUSTER) > $(KIND_KUBECONFIG); \ else \ echo "Creating kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER); \ + $(KIND) create cluster --name $(KIND_CLUSTER) --kubeconfig $(KIND_KUBECONFIG); \ fi - @echo "==> Exporting kubeconfig to $(KIND_KUBECONFIG)" - @$(KIND) get kubeconfig --name $(KIND_CLUSTER) > $(KIND_KUBECONFIG) @echo "==> Cluster ready. Use: export KUBECONFIG=$(KIND_KUBECONFIG)" .PHONY: kind-load @@ -430,7 +430,7 @@ kind-deploy: kind-up manifests kustomize kind-load ## Deploy operator to kind cl .PHONY: kind-redeploy kind-redeploy: kind-load ## Rebuild image, reload to kind, and restart pods @echo "==> Restarting operator pods..." - KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator-system + KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator .PHONY: kind-down kind-down: ## Delete the kind cluster diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 80567b94..03b86f2b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/numtide/multigres-operator - newTag: e2b8632-dirty + newTag: a96d950 diff --git a/pkg/resource-handler/controller/cell/integration_test.go b/pkg/resource-handler/controller/cell/integration_test.go index b4a16e52..67cefe80 100644 --- a/pkg/resource-handler/controller/cell/integration_test.go +++ b/pkg/resource-handler/controller/cell/integration_test.go @@ -114,7 +114,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Ports: []corev1.ContainerPort{ @@ -201,7 +200,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", }, Ports: []corev1.ContainerPort{ @@ -288,7 +286,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone3", }, Ports: []corev1.ContainerPort{ @@ -392,7 +389,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone4", }, Ports: []corev1.ContainerPort{ diff --git a/pkg/resource-handler/controller/cell/multigateway.go b/pkg/resource-handler/controller/cell/multigateway.go index d5d0110b..ca918cc7 100644 --- a/pkg/resource-handler/controller/cell/multigateway.go +++ b/pkg/resource-handler/controller/cell/multigateway.go @@ -79,7 +79,6 @@ func BuildMultiGatewayDeployment( "--pg-port", fmt.Sprintf("%d", MultiGatewayPostgresPort), "--topo-global-server-addresses", cell.Spec.GlobalTopoServer.Address, "--topo-global-root", cell.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", cell.Spec.GlobalTopoServer.Implementation, "--cell", cell.Spec.Name, }, Resources: cell.Spec.MultiGateway.Resources, diff --git a/pkg/resource-handler/controller/cell/multigateway_test.go b/pkg/resource-handler/controller/cell/multigateway_test.go index 013361ae..f9eea787 100644 --- a/pkg/resource-handler/controller/cell/multigateway_test.go +++ b/pkg/resource-handler/controller/cell/multigateway_test.go @@ -97,7 +97,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Resources: corev1.ResourceRequirements{}, @@ -200,7 +199,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", }, Resources: corev1.ResourceRequirements{}, @@ -303,7 +301,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone3", }, Resources: corev1.ResourceRequirements{}, @@ -422,7 +419,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone4", }, Resources: corev1.ResourceRequirements{}, @@ -551,7 +547,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone5", }, Resources: corev1.ResourceRequirements{ diff --git a/pkg/resource-handler/controller/shard/configmap.go b/pkg/resource-handler/controller/shard/configmap.go new file mode 100644 index 00000000..12c51673 --- /dev/null +++ b/pkg/resource-handler/controller/shard/configmap.go @@ -0,0 +1,55 @@ +package shard + +import ( + _ "embed" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + + multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" +) + +// DefaultPgHbaTemplate is the default pg_hba.conf template for pooler instances. +// Uses trust authentication for testing/development. Production deployments should +// override this with proper authentication (scram-sha-256, SSL certificates, etc.). +// +//go:embed templates/pg_hba_template.conf +var DefaultPgHbaTemplate string + +// BuildPgHbaConfigMap creates a ConfigMap containing the pg_hba.conf template. +// This ConfigMap is shared across all pools in a shard and mounted into postgres containers. +func BuildPgHbaConfigMap( + shard *multigresv1alpha1.Shard, + scheme *runtime.Scheme, +) (*corev1.ConfigMap, error) { + // TODO: Add Shard.Spec.PgHbaTemplate field to allow custom templates + template := DefaultPgHbaTemplate + + labels := map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": shard.Name, + "app.kubernetes.io/component": "pg-hba-config", + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: PgHbaConfigMapName, + Namespace: shard.Namespace, + Labels: labels, + }, + Data: map[string]string{ + "pg_hba_template.conf": template, + }, + } + + if err := ctrl.SetControllerReference(shard, cm, scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + + return cm, nil +} diff --git a/pkg/resource-handler/controller/shard/configmap_test.go b/pkg/resource-handler/controller/shard/configmap_test.go new file mode 100644 index 00000000..e3f40577 --- /dev/null +++ b/pkg/resource-handler/controller/shard/configmap_test.go @@ -0,0 +1,172 @@ +package shard + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" +) + +func TestBuildPgHbaConfigMap(t *testing.T) { + defaultScheme := runtime.NewScheme() + _ = multigresv1alpha1.AddToScheme(defaultScheme) + + tests := map[string]struct { + shard *multigresv1alpha1.Shard + scheme *runtime.Scheme + wantErr bool + }{ + "creates configmap with default template": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + UID: "test-uid", + }, + }, + wantErr: false, + }, + "creates configmap with correct embedded template": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "production-shard", + Namespace: "prod", + UID: "prod-uid", + }, + }, + wantErr: false, + }, + "returns error when scheme is invalid (missing Shard kind)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "error-shard", + Namespace: "default", + UID: "error-uid", + }, + }, + scheme: runtime.NewScheme(), // Empty scheme + wantErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + scheme := tc.scheme + if scheme == nil { + scheme = defaultScheme + } + + cm, err := BuildPgHbaConfigMap(tc.shard, scheme) + if (err != nil) != tc.wantErr { + t.Fatalf("BuildPgHbaConfigMap() error = %v, wantErr %v", err, tc.wantErr) + } + + if tc.wantErr { + return + } + + if cm.Name != PgHbaConfigMapName { + t.Errorf("ConfigMap name = %v, want %v", cm.Name, PgHbaConfigMapName) + } + if cm.Namespace != tc.shard.Namespace { + t.Errorf("ConfigMap namespace = %v, want %v", cm.Namespace, tc.shard.Namespace) + } + + // Verify owner reference + if len(cm.OwnerReferences) != 1 { + t.Fatalf("Expected 1 owner reference, got %d", len(cm.OwnerReferences)) + } + ownerRef := cm.OwnerReferences[0] + if ownerRef.Name != tc.shard.Name || ownerRef.Kind != "Shard" { + t.Errorf("Owner reference = %+v, want Shard/%s", ownerRef, tc.shard.Name) + } + if !ptr.Deref(ownerRef.Controller, false) { + t.Error("Expected owner reference to be controller") + } + + // Verify labels + expectedLabels := map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": tc.shard.Name, + "app.kubernetes.io/component": "pg-hba-config", + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + } + if diff := cmp.Diff(expectedLabels, cm.Labels); diff != "" { + t.Errorf("Labels mismatch (-want +got):\n%s", diff) + } + + // Verify template content exists + template, ok := cm.Data["pg_hba_template.conf"] + if !ok { + t.Fatal("ConfigMap missing pg_hba_template.conf key") + } + + // Verify the template matches what's embedded (source of truth) + if template != DefaultPgHbaTemplate { + t.Error("Template content doesn't match DefaultPgHbaTemplate") + } + }) + } +} + +func TestDefaultPgHbaTemplateEmbedded(t *testing.T) { + // Verify the embedded template is not empty + if DefaultPgHbaTemplate == "" { + t.Error("DefaultPgHbaTemplate is empty - go:embed may have failed") + } + + // Verify critical configuration lines exist + // We check for the presence of rules, ignoring multiple spaces + checks := []struct { + desc string + mustContain []string + }{ + { + desc: "header", + mustContain: []string{"# PostgreSQL Client Authentication"}, + }, + { + desc: "local trust rule", + mustContain: []string{"local", "all", "{{.User}}", "trust"}, + }, + { + desc: "replication trust rule", + mustContain: []string{"host", "replication", "all", "0.0.0.0/0", "trust"}, + }, + { + desc: "production warning", + mustContain: []string{"PRODUCTION:", "scram-sha-256"}, + }, + } + + for _, check := range checks { + found := false + lines := strings.Split(DefaultPgHbaTemplate, "\n") + for _, line := range lines { + allMatch := true + for _, part := range check.mustContain { + if !strings.Contains(line, part) { + allMatch = false + break + } + } + if allMatch { + found = true + break + } + } + if !found { + t.Errorf( + "DefaultPgHbaTemplate missing %s (expected line containing all of: %v)", + check.desc, + check.mustContain, + ) + } + } +} diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index d93f6a60..0b12f878 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -1,39 +1,116 @@ package shard import ( - "fmt" - corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" ) const ( - // DefaultMultigresImage is the base image for all Multigres components (multipooler, multiorch, pgctld) + // DefaultMultigresImage is the base image for all Multigres components (multipooler, multiorch) // Different components use different subcommands. DefaultMultigresImage = "ghcr.io/multigres/multigres:main" + // DefaultPgctldImage is the image containing the pgctld binary + // Used by buildPgctldInitContainer() for the binary-copy approach + DefaultPgctldImage = "ghcr.io/multigres/pgctld:main" + // DefaultPostgresImage is the default PostgreSQL database container image + // Used by buildPostgresContainer() for the original stock postgres:17 approach + // NOTE: Currently unused - buildPgctldContainer() uses ghcr.io/multigres/pgctld:main instead DefaultPostgresImage = "postgres:17" // PgctldVolumeName is the name of the shared volume for pgctld binary + // Used only by alternative approach (binary-copy via init container) PgctldVolumeName = "pgctld-bin" - // PgctldMountPath is the mount path for pgctld binary in postgres container - PgctldMountPath = "/usr/local/bin/pgctld" + // PgctldBinDir is the directory where pgctld binary is mounted + // Subdirectory avoids shadowing postgres binaries in /usr/local/bin + // Used only by alternative approach (binary-copy via init container) + PgctldBinDir = "/usr/local/bin/multigres" + + // PgctldMountPath is the full path to pgctld binary + // Used only by alternative approach (binary-copy via init container) + PgctldMountPath = PgctldBinDir + "/pgctld" // DataVolumeName is the name of the data volume for PostgreSQL DataVolumeName = "pgdata" - // DataMountPath is the mount path for PostgreSQL data - DataMountPath = "/var/lib/postgresql/data" + // DataMountPath is where the PVC is mounted + // Mounted at parent directory because mounting directly at pg_data/ prevents + // initdb from setting directory permissions (non-root can't chmod mount points). + // pgctld creates pg_data/ subdirectory with proper 0700/0750 permissions. + DataMountPath = "/var/lib/pooler" + + // PgDataPath is the actual postgres data directory (PGDATA env var value) + // pgctld expects postgres data at /pg_data + PgDataPath = "/var/lib/pooler/pg_data" + + // PoolerDirMountPath must equal DataMountPath because both containers share the PVC + // and pgctld derives postgres data directory as /pg_data + PoolerDirMountPath = "/var/lib/pooler" + + // SocketDirVolumeName is the name of the shared volume for unix sockets + SocketDirVolumeName = "socket-dir" + + // SocketDirMountPath is the mount path for unix sockets (postgres and pgctld communicate here) + // We use /var/run/postgresql because that is the default socket directory for the official postgres image. + SocketDirMountPath = "/var/run/postgresql" + + // BackupVolumeName is the name of the backup volume for pgbackrest + BackupVolumeName = "backup-data" + + // BackupMountPath is where the backup volume is mounted + // pgbackrest stores backups here via --repo1-path + BackupMountPath = "/backups" + + // PgHbaConfigMapName is the name of the ConfigMap containing pg_hba template + PgHbaConfigMapName = "pg-hba-template" + + // PgHbaVolumeName is the name of the volume for pg_hba template + PgHbaVolumeName = "pg-hba-template" + + // PgHbaMountPath is where the pg_hba template is mounted + PgHbaMountPath = "/etc/pgctld" + + // PgHbaTemplatePath is the full path to the pg_hba template file + PgHbaTemplatePath = PgHbaMountPath + "/pg_hba_template.conf" ) +// buildSocketDirVolume creates the shared emptyDir volume for unix sockets. +func buildSocketDirVolume() corev1.Volume { + return corev1.Volume{ + Name: SocketDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + +// buildPgHbaVolume creates the volume for pg_hba template from ConfigMap. +func buildPgHbaVolume() corev1.Volume { + return corev1.Volume{ + Name: PgHbaVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: PgHbaConfigMapName, + }, + }, + }, + } +} + // sidecarRestartPolicy is the restart policy for native sidecar containers var sidecarRestartPolicy = corev1.ContainerRestartPolicyAlways // buildPostgresContainer creates the postgres container spec for a pool. -// This runs pgctld binary (which wraps postgres) and mounts persistent data storage. +// Uses stock postgres:17 image with pgctld and pgbackrest binaries copied via init container. +// +// This approach requires: +// - buildPgctldInitContainer() in InitContainers (copies pgctld and pgbackrest) +// - buildPgctldVolume() in Volumes func buildPostgresContainer( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -44,9 +121,34 @@ func buildPostgresContainer( } return corev1.Container{ - Name: "postgres", - Image: image, + Name: "postgres", + Image: image, + Command: []string{PgctldMountPath}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, + }, Resources: pool.Postgres.Resources, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), // Must match postgres:17 image UID for file access + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), // pgctld refuses to run as root + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -54,17 +156,97 @@ func buildPostgresContainer( }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, + }, + } +} + +// buildPgctldContainer creates the postgres container spec using the pgctld image. +// Uses DefaultPgctldImage (ghcr.io/multigres/pgctld:main) which includes: +// - PostgreSQL 17 +// - pgctld binary at /usr/local/bin/pgctld +// - pgbackrest for backup/restore operations +// +// This approach does NOT require: +// - buildPgctldInitContainer() (pgctld already in image) +// - buildPgctldVolume() (no binary copying needed) +func buildPgctldContainer( + shard *multigresv1alpha1.Shard, + pool multigresv1alpha1.PoolSpec, +) corev1.Container { + image := DefaultPgctldImage + if shard.Spec.Images.Postgres != "" { + image = shard.Spec.Images.Postgres + } + + return corev1.Container{ + Name: "postgres", + Image: image, + Command: []string{"/usr/local/bin/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, + }, + Resources: pool.Postgres.Resources, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: DataMountPath, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, } } // buildMultiPoolerSidecar creates the multipooler sidecar container spec. -// This is implemented as a native sidecar using init container with -// restartPolicy: Always (K8s 1.28+). -// cellName specifies which cell this container is running in. -// If cellName is empty, defaults to the global topology cell. +// Implemented as native sidecar (init container with restartPolicy: Always) because +// multipooler must restart with postgres to maintain connection pool consistency. func buildMultiPoolerSidecar( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -77,21 +259,23 @@ func buildMultiPoolerSidecar( } // TODO: Add remaining command line arguments: - // --pooler-dir, --grpc-socket-file, --log-level, --log-output, --hostname, --service-map - // --pgbackrest-stanza, --connpool-admin-password, --socket-file + // --grpc-socket-file, --log-level, --log-output, --hostname + // --pgbackrest-stanza, --connpool-admin-password args := []string{ "multipooler", // Subcommand "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", // Unix socket uses trust auth (no password) + "--service-map", "grpc-pooler", // Only enable grpc-pooler service (disables auto-restore service) "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", shard.Spec.GlobalTopoServer.Implementation, "--cell", cellName, "--database", shard.Spec.DatabaseName, "--table-group", shard.Spec.TableGroupName, "--shard", shard.Spec.ShardName, - "--service-id", getPoolServiceID(shard.Name, poolName), + "--service-id", "$(POD_NAME)", // Use pod name as unique service ID "--pgctld-addr", "localhost:15470", "--pg-port", "5432", } @@ -103,22 +287,48 @@ func buildMultiPoolerSidecar( Ports: buildMultiPoolerContainerPorts(), Resources: pool.Multipooler.Resources, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), // Must match postgres UID to access pg_data directory + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, // Shares PVC with postgres for pgbackrest configs and sockets + MountPath: PoolerDirMountPath, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + }, } } // buildPgctldInitContainer creates the pgctld init container spec. -// This copies the pgctld binary to a shared volume for use by the postgres container. +// Copies pgctld and pgbackrest binaries to shared volume for use with stock postgres:17 image. +// Used with buildPostgresContainer() and buildPgctldVolume(). func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { - image := DefaultMultigresImage - // TODO: Add pgctld image field to Shard spec if needed - return corev1.Container{ - Name: "pgctld-init", - Image: image, + Name: "pgctld-init", + Image: DefaultPgctldImage, + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", // Subcommand - "copy-binary", - "--output", "/shared/pgctld", + "cp /usr/local/bin/pgctld /usr/bin/pgbackrest /shared/", }, VolumeMounts: []corev1.VolumeMount{ { @@ -137,17 +347,24 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } // TODO: Add remaining command line arguments: - // --watch-targets, --log-level, --log-output, --hostname - // --cluster-metadata-refresh-interval, --pooler-health-check-interval, --recovery-cycle-interval + // --log-level, --log-output, --hostname + // TODO: Verify correct format for --watch-targets flag. + // Currently using static "postgres" based on demo, but may need to be: + // - Just database name (e.g., "postgres") + // - Full path (e.g., "database/tablegroup/shard") + // - Multiple targets (e.g., "postgres,otherdb") args := []string{ "multiorch", // Subcommand "--http-port", "15300", "--grpc-port", "15370", "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", shard.Spec.GlobalTopoServer.Implementation, "--cell", cellName, + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", } return corev1.Container{ @@ -159,7 +376,8 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } } -// buildPgctldVolume creates the shared emptyDir volume for pgctld binary. +// buildPgctldVolume creates the shared emptyDir volume for pgctld and pgbackrest binaries. +// Used with buildPgctldInitContainer() and buildPostgresContainer(). func buildPgctldVolume() corev1.Volume { return corev1.Volume{ Name: PgctldVolumeName, @@ -169,10 +387,16 @@ func buildPgctldVolume() corev1.Volume { } } -// getPoolServiceID generates a unique service ID for a pool. -// This is used in multipooler and pgctld arguments. -func getPoolServiceID(shardName string, poolName string) string { - // TODO: Use proper ID generation (UUID or consistent hash) - // For now, use simple format - return fmt.Sprintf("%s-pool-%s", shardName, poolName) +// buildBackupVolume creates the backup volume for pgbackrest. +// References a PVC that is created separately and shared across all pods in a pool. +// For single-node clusters (kind), ReadWriteOnce works since all pods are on the same node. +func buildBackupVolume(poolName string) corev1.Volume { + return corev1.Volume{ + Name: BackupVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-" + poolName, + }, + }, + } } diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 6f37080a..5bf87389 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" ) @@ -23,9 +24,34 @@ func TestBuildPostgresContainer(t *testing.T) { }, poolSpec: multigresv1alpha1.PoolSpec{}, want: corev1.Container{ - Name: "postgres", - Image: DefaultPostgresImage, + Name: "postgres", + Image: DefaultPostgresImage, + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, + }, Resources: corev1.ResourceRequirements{}, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -33,7 +59,20 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -48,9 +87,34 @@ func TestBuildPostgresContainer(t *testing.T) { }, poolSpec: multigresv1alpha1.PoolSpec{}, want: corev1.Container{ - Name: "postgres", - Image: "postgres:16", + Name: "postgres", + Image: "postgres:16", + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, + }, Resources: corev1.ResourceRequirements{}, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -58,7 +122,20 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -82,8 +159,22 @@ func TestBuildPostgresContainer(t *testing.T) { }, }, want: corev1.Container{ - Name: "postgres", - Image: DefaultPostgresImage, + Name: "postgres", + Image: DefaultPostgresImage, + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, + }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), @@ -94,6 +185,17 @@ func TestBuildPostgresContainer(t *testing.T) { corev1.ResourceMemory: resource.MustParse("4Gi"), }, }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -101,7 +203,20 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -151,20 +266,51 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "test-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, Ports: buildMultiPoolerContainerPorts(), Resources: corev1.ResourceRequirements{}, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + }, }, }, "custom multipooler image": { @@ -195,20 +341,51 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", "--database", "proddb", "--table-group", "orders", "--shard", "1", - "--service-id", "custom-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, Ports: buildMultiPoolerContainerPorts(), Resources: corev1.ResourceRequirements{}, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + }, }, }, "with resource requirements": { @@ -248,14 +425,16 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "mydb", "--table-group", "default", "--shard", "0", - "--service-id", "resource-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -271,6 +450,35 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { }, }, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + }, }, }, } @@ -296,12 +504,11 @@ func TestBuildPgctldInitContainer(t *testing.T) { Spec: multigresv1alpha1.ShardSpec{}, }, want: corev1.Container{ - Name: "pgctld-init", - Image: DefaultMultigresImage, + Name: "pgctld-init", + Image: DefaultPgctldImage, + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + "cp /usr/local/bin/pgctld /usr/bin/pgbackrest /shared/", }, VolumeMounts: []corev1.VolumeMount{ { @@ -350,8 +557,11 @@ func TestBuildMultiOrchContainer(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: buildMultiOrchContainerPorts(), Resources: corev1.ResourceRequirements{}, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 6aa95c5e..55efa795 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -79,11 +79,11 @@ func TestShardReconciliation(t *testing.T) { Implementation: "etcd2", }, MultiOrch: multigresv1alpha1.MultiOrchSpec{ - Cells: []multigresv1alpha1.CellName{"us-west-1a", "us-west-1b"}, + Cells: []multigresv1alpha1.CellName{"zone-a", "zone-b"}, }, Pools: map[string]multigresv1alpha1.PoolSpec{ "primary": { - Cells: []multigresv1alpha1.CellName{"us-west-1a"}, + Cells: []multigresv1alpha1.CellName{"zone-a"}, Type: "readWrite", ReplicasPerCell: ptr.To(int32(2)), Storage: multigresv1alpha1.StorageSpec{ @@ -94,22 +94,22 @@ func TestShardReconciliation(t *testing.T) { }, }, wantResources: []client.Object{ - // MultiOrch Deployment for us-west-1a + // MultiOrch Deployment for zone-a &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.DeploymentSpec{ Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + MatchLabels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -122,8 +122,11 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", - "--cell", "us-west-1a", + "--cell", "zone-a", + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -135,12 +138,12 @@ func TestShardReconciliation(t *testing.T) { }, }, }, - // MultiOrch Service for us-west-1a + // MultiOrch Service for zone-a &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -149,25 +152,25 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "http", 15300), tcpServicePort(t, "grpc", 15370), }, - Selector: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Selector: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, }, - // MultiOrch Deployment for us-west-1b + // MultiOrch Deployment for zone-b &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1b", + Name: "test-shard-multiorch-zone-b", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.DeploymentSpec{ Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + MatchLabels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -180,8 +183,11 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", - "--cell", "us-west-1b", + "--cell", "zone-b", + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -193,12 +199,12 @@ func TestShardReconciliation(t *testing.T) { }, }, }, - // MultiOrch Service for us-west-1b + // MultiOrch Service for zone-b &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1b", + Name: "test-shard-multiorch-zone-b", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -207,21 +213,21 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "http", 15300), tcpServicePort(t, "grpc", 15370), }, - Selector: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Selector: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, }, &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-pool-primary-us-west-1a", + Name: "test-shard-pool-primary-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.StatefulSetSpec{ - ServiceName: "test-shard-pool-primary-us-west-1a-headless", + ServiceName: "test-shard-pool-primary-zone-a-headless", Replicas: ptr.To(int32(2)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + MatchLabels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), }, PodManagementPolicy: appsv1.ParallelPodManagement, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ @@ -229,22 +235,22 @@ func TestShardReconciliation(t *testing.T) { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), }, Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ - { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", - Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, - }, - }, + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-init", + // Image: "ghcr.io/multigres/pgctld:main", + // Command: []string{"/bin/sh", "-c"}, + // Args: []string{ + // "cp /usr/local/bin/pgctld /shared/pgctld", + // }, + // VolumeMounts: []corev1.VolumeMount{ + // {Name: "pgctld-bin", MountPath: "/shared"}, + // }, + // }, { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -252,38 +258,113 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", - "--cell", "us-west-1a", + "--cell", "zone-a", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "test-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, Ports: multipoolerPorts(t), RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, Volumes: []corev1.Volume{ + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, + { + Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-test-shard-pool-primary-zone-a", + }, + }, + }, { - Name: "pgctld-bin", + Name: "socket-dir", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, + }, + }, }, }, }, @@ -310,9 +391,9 @@ func TestShardReconciliation(t *testing.T) { }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-pool-primary-us-west-1a-headless", + Name: "test-shard-pool-primary-zone-a-headless", Namespace: "default", - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -323,7 +404,7 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "grpc", 15270), tcpServicePort(t, "postgres", 5432), }, - Selector: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Selector: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), PublishNotReadyAddresses: true, }, }, @@ -393,8 +474,11 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -451,8 +535,11 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", + "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -500,27 +587,44 @@ func TestShardReconciliation(t *testing.T) { Labels: shardLabels(t, "multi-cell-shard-pool-primary-zone1", "shard-pool", "zone1"), }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, Volumes: []corev1.Volume{ + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, { - Name: "pgctld-bin", + Name: "backup-data", VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-multi-cell-shard-pool-primary-zone1", + }, }, }, - }, - InitContainers: []corev1.Container{ { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", - Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + Name: "socket-dir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, + }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, }, }, + }, + InitContainers: []corev1.Container{ { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -528,14 +632,16 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "multi-cell-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -545,15 +651,62 @@ func TestShardReconciliation(t *testing.T) { tcpPort(t, "postgres", 5432), }, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, @@ -619,27 +772,44 @@ func TestShardReconciliation(t *testing.T) { Labels: shardLabels(t, "multi-cell-shard-pool-primary-zone2", "shard-pool", "zone2"), }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, Volumes: []corev1.Volume{ + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, { - Name: "pgctld-bin", + Name: "backup-data", VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-multi-cell-shard-pool-primary-zone2", + }, }, }, - }, - InitContainers: []corev1.Container{ { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", - Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + Name: "socket-dir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, + }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, }, }, + }, + InitContainers: []corev1.Container{ { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -647,14 +817,16 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "multi-cell-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -664,15 +836,62 @@ func TestShardReconciliation(t *testing.T) { tcpPort(t, "postgres", 5432), }, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, diff --git a/pkg/resource-handler/controller/shard/multiorch_test.go b/pkg/resource-handler/controller/shard/multiorch_test.go index e430eb60..88c7ad31 100644 --- a/pkg/resource-handler/controller/shard/multiorch_test.go +++ b/pkg/resource-handler/controller/shard/multiorch_test.go @@ -41,23 +41,23 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Implementation: "etcd2", }, MultiOrch: multigresv1alpha1.MultiOrchSpec{ - Cells: []multigresv1alpha1.CellName{"us-west-1a"}, + Cells: []multigresv1alpha1.CellName{"zone-a"}, }, }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: scheme, want: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -77,11 +77,11 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -90,11 +90,11 @@ func TestBuildMultiOrchDeployment(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -109,7 +109,7 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Implementation: "etcd2", }, }, - }, "us-west-1a"), + }, "zone-a"), }, }, }, @@ -315,7 +315,7 @@ func TestBuildMultiOrchDeployment(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: runtime.NewScheme(), // empty scheme wantErr: true, }, @@ -364,19 +364,19 @@ func TestBuildMultiOrchService(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: scheme, want: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -395,11 +395,11 @@ func TestBuildMultiOrchService(t *testing.T) { Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -499,7 +499,7 @@ func TestBuildMultiOrchService(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: runtime.NewScheme(), // empty scheme wantErr: true, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 5fcc7b94..d6afb104 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -5,8 +5,10 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" @@ -67,18 +69,25 @@ func BuildPoolStatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ - // Init containers: pgctld copies binary, multipooler is a native sidecar + // FSGroup ensures PVC is writable by postgres user (required for initdb) + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image + }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(shard), + // To use stock postgres:17 image instead, uncomment buildPgctldInitContainer, + // replace buildPgctldContainer with buildPostgresContainer, and add buildPgctldVolume. + // buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, - // Postgres is the main container (runs pgctld binary) Containers: []corev1.Container{ - buildPostgresContainer(shard, poolSpec), + buildPgctldContainer(shard, poolSpec), + // buildPostgresContainer(shard, poolSpec), }, - // Shared volume for pgctld binary Volumes: []corev1.Volume{ - buildPgctldVolume(), + // buildPgctldVolume(), + buildBackupVolume(name), + buildSocketDirVolume(), + buildPgHbaVolume(), }, Affinity: poolSpec.Affinity, }, @@ -110,6 +119,76 @@ func buildPoolVolumeClaimTemplates( } return []corev1.PersistentVolumeClaim{ - storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize), + storage.BuildPVCTemplate( + DataVolumeName, + storageClass, + storageSize, + pool.Storage.AccessModes, + ), + } +} + +// BuildBackupPVC creates a standalone PVC for backup storage shared across all pods in a pool. +// This PVC is created independently of the StatefulSet and referenced by all pods. +// For single-node clusters (kind, minikube), uses ReadWriteOnce (all pods on same node). +// For multi-node production, configure BackupStorage.Class to a storage class supporting ReadWriteMany. +func BuildBackupPVC( + shard *multigresv1alpha1.Shard, + poolName string, + cellName string, + poolSpec multigresv1alpha1.PoolSpec, + scheme *runtime.Scheme, +) (*corev1.PersistentVolumeClaim, error) { + name := buildPoolNameWithCell(shard.Name, poolName, cellName) + pvcName := "backup-data-" + name + labels := buildPoolLabelsWithCell(shard, poolName, cellName, poolSpec) + + // Use BackupStorage if specified, otherwise inherit from Storage + var storageClass *string + storageSize := "10Gi" // Default backup storage size + + if poolSpec.BackupStorage.Class != "" { + storageClass = &poolSpec.BackupStorage.Class + } else if poolSpec.Storage.Class != "" { + storageClass = &poolSpec.Storage.Class + } + + if poolSpec.BackupStorage.Size != "" { + storageSize = poolSpec.BackupStorage.Size + } + + // Default to ReadWriteOnce for single-node clusters. + accessModes := []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, } + + if len(poolSpec.BackupStorage.AccessModes) > 0 { + accessModes = poolSpec.BackupStorage.AccessModes + } + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: shard.Namespace, + Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: accessModes, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(storageSize), + }, + }, + }, + } + + if storageClass != nil { + pvc.Spec.StorageClassName = storageClass + } + + if err := ctrl.SetControllerReference(shard, pvc, scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + + return pvc, nil } diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 5322f756..61396012 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -105,13 +105,10 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "test-shard"}, @@ -120,24 +117,72 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, "primary", "zone1", ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", + }, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - multigresv1alpha1.PoolSpec{}, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildPgctldVolume(), + buildBackupVolume("test-shard-pool-primary-zone1"), + buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -242,13 +287,10 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-001"}, @@ -258,25 +300,74 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, multigresv1alpha1.PoolSpec{ - Cells: []multigresv1alpha1.CellName{"zone-west"}, + Type: "readOnly", + Cells: []multigresv1alpha1.CellName{"zone-west"}, + ReplicasPerCell: ptr.To(int32(3)), + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + Size: "20Gi", + }, }, "replica", "zone-west", ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", + }, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - multigresv1alpha1.PoolSpec{}, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildPgctldVolume(), + buildBackupVolume("shard-001-pool-replica-zone-west"), + buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -378,13 +469,10 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-002"}, @@ -393,24 +481,71 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Size: "5Gi", + }, + }, "readOnly", "zone1", ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", + }, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - multigresv1alpha1.PoolSpec{}, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildPgctldVolume(), + buildBackupVolume("shard-002-pool-readOnly-zone1"), + buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -529,13 +664,10 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-affinity"}, @@ -544,24 +676,89 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "disk-type", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"ssd"}, + }, + }, + }, + }, + }, + }, + }, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, "primary", "zone1", ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", + }, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - multigresv1alpha1.PoolSpec{}, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildPgctldVolume(), + buildBackupVolume("shard-affinity-pool-primary-zone1"), + buildSocketDirVolume(), + buildPgHbaVolume(), }, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ @@ -734,3 +931,199 @@ func TestBuildPoolVolumeClaimTemplates(t *testing.T) { }) } } + +func TestBuildBackupPVC(t *testing.T) { + scheme := runtime.NewScheme() + _ = multigresv1alpha1.AddToScheme(scheme) + + shard := &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + UID: "test-uid", + }, + } + + tests := map[string]struct { + poolSpec multigresv1alpha1.PoolSpec + want *corev1.PersistentVolumeClaim + }{ + "defaults": { + poolSpec: multigresv1alpha1.PoolSpec{}, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + "inherit storage class": { + poolSpec: multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + }, + }, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("fast-ssd"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + "override backup storage": { + poolSpec: multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + }, + BackupStorage: multigresv1alpha1.StorageSpec{ + Class: "backup-nfs", + Size: "100Gi", + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + }, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("backup-nfs"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("100Gi"), + }, + }, + }, + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + testScheme := scheme + if name == "error on controller reference" { + testScheme = runtime.NewScheme() // Empty scheme triggers SetControllerReference error + } + + got, err := BuildBackupPVC( + shard, + "primary", + "zone1", + tc.poolSpec, + testScheme, + ) + + if name == "error on controller reference" { + if err == nil { + t.Error("BuildBackupPVC() expected error, got nil") + } + return + } + + if err != nil { + t.Errorf("BuildBackupPVC() error = %v", err) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("BuildBackupPVC() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestBuildBackupPVC_Error(t *testing.T) { + // Separate test for error case to avoid messing with table driven test logic too much + shard := &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + } + // Empty scheme causes SetControllerReference to fail + _, err := BuildBackupPVC( + shard, + "pool", + "cell", + multigresv1alpha1.PoolSpec{}, + runtime.NewScheme(), + ) + if err == nil { + t.Error("BuildBackupPVC() expected error with empty scheme, got nil") + } +} diff --git a/pkg/resource-handler/controller/shard/shard_controller.go b/pkg/resource-handler/controller/shard/shard_controller.go index f8498838..7b8bda75 100644 --- a/pkg/resource-handler/controller/shard/shard_controller.go +++ b/pkg/resource-handler/controller/shard/shard_controller.go @@ -60,6 +60,12 @@ func (r *ShardReconciler) Reconcile( } } + // Reconcile pg_hba ConfigMap first (required by all pools before StatefulSets start) + if err := r.reconcilePgHbaConfigMap(ctx, shard); err != nil { + logger.Error(err, "Failed to reconcile pg_hba ConfigMap") + return ctrl.Result{}, err + } + // Reconcile MultiOrch - one Deployment and Service per cell multiOrchCells, err := getMultiOrchCells(shard) if err != nil { @@ -161,6 +167,44 @@ func (r *ShardReconciler) reconcileMultiOrchDeployment( return nil } +// reconcilePgHbaConfigMap creates or updates the pg_hba ConfigMap for a shard. +// This ConfigMap is shared across all pools and contains the authentication template. +func (r *ShardReconciler) reconcilePgHbaConfigMap( + ctx context.Context, + shard *multigresv1alpha1.Shard, +) error { + desired, err := BuildPgHbaConfigMap(shard, r.Scheme) + if err != nil { + return fmt.Errorf("failed to build pg_hba ConfigMap: %w", err) + } + + existing := &corev1.ConfigMap{} + err = r.Get( + ctx, + client.ObjectKey{Namespace: shard.Namespace, Name: desired.Name}, + existing, + ) + if err != nil { + if errors.IsNotFound(err) { + // Create new ConfigMap + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create pg_hba ConfigMap: %w", err) + } + return nil + } + return fmt.Errorf("failed to get pg_hba ConfigMap: %w", err) + } + + // Update existing ConfigMap + existing.Data = desired.Data + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update pg_hba ConfigMap: %w", err) + } + + return nil +} + // reconcileMultiOrchService creates or updates the MultiOrch Service for a specific cell. func (r *ShardReconciler) reconcileMultiOrchService( ctx context.Context, @@ -221,6 +265,11 @@ func (r *ShardReconciler) reconcilePool( for _, cell := range poolSpec.Cells { cellName := string(cell) + // Reconcile backup PVC before StatefulSet (PVC must exist first) + if err := r.reconcilePoolBackupPVC(ctx, shard, poolName, cellName, poolSpec); err != nil { + return fmt.Errorf("failed to reconcile backup PVC for cell %s: %w", cellName, err) + } + // Reconcile pool StatefulSet for this cell if err := r.reconcilePoolStatefulSet(ctx, shard, poolName, cellName, poolSpec); err != nil { return fmt.Errorf("failed to reconcile pool StatefulSet for cell %s: %w", cellName, err) @@ -279,6 +328,47 @@ func (r *ShardReconciler) reconcilePoolStatefulSet( return nil } +// reconcilePoolBackupPVC creates or updates the shared backup PVC for a pool in a specific cell. +func (r *ShardReconciler) reconcilePoolBackupPVC( + ctx context.Context, + shard *multigresv1alpha1.Shard, + poolName string, + cellName string, + poolSpec multigresv1alpha1.PoolSpec, +) error { + desired, err := BuildBackupPVC(shard, poolName, cellName, poolSpec, r.Scheme) + if err != nil { + return fmt.Errorf("failed to build backup PVC: %w", err) + } + + existing := &corev1.PersistentVolumeClaim{} + err = r.Get( + ctx, + client.ObjectKey{Namespace: shard.Namespace, Name: desired.Name}, + existing, + ) + if err != nil { + if errors.IsNotFound(err) { + // Create new PVC + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create backup PVC: %w", err) + } + return nil + } + return fmt.Errorf("failed to get backup PVC: %w", err) + } + + // PVCs are immutable after creation, only update labels/annotations if needed + if desired.Labels != nil { + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update backup PVC labels: %w", err) + } + } + + return nil +} + // reconcilePoolHeadlessService creates or updates the headless Service for a pool in a specific cell. func (r *ShardReconciler) reconcilePoolHeadlessService( ctx context.Context, diff --git a/pkg/resource-handler/controller/shard/shard_controller_internal_test.go b/pkg/resource-handler/controller/shard/shard_controller_internal_test.go index 6bdc97f2..36cb57b7 100644 --- a/pkg/resource-handler/controller/shard/shard_controller_internal_test.go +++ b/pkg/resource-handler/controller/shard/shard_controller_internal_test.go @@ -188,6 +188,22 @@ func TestReconcile_InvalidScheme(t *testing.T) { return r.reconcilePoolHeadlessService(ctx, shard, "pool1", "", poolSpec) }, }, + "PoolBackupPVC": { + setupShard: func() *multigresv1alpha1.Shard { + return &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + }, + } + }, + reconcileFunc: func(r *ShardReconciler, ctx context.Context, shard *multigresv1alpha1.Shard) error { + poolSpec := multigresv1alpha1.PoolSpec{ + Cells: []multigresv1alpha1.CellName{"cell1"}, + } + return r.reconcilePoolBackupPVC(ctx, shard, "pool1", "cell1", poolSpec) + }, + }, } for name, tc := range tests { diff --git a/pkg/resource-handler/controller/shard/shard_controller_test.go b/pkg/resource-handler/controller/shard/shard_controller_test.go index 46cca7df..50067d5d 100644 --- a/pkg/resource-handler/controller/shard/shard_controller_test.go +++ b/pkg/resource-handler/controller/shard/shard_controller_test.go @@ -28,11 +28,12 @@ func TestShardReconciler_Reconcile(t *testing.T) { _ = corev1.AddToScheme(scheme) tests := map[string]struct { - shard *multigresv1alpha1.Shard - existingObjects []client.Object - failureConfig *testutil.FailureConfig - wantErr bool - assertFunc func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) + shard *multigresv1alpha1.Shard + existingObjects []client.Object + failureConfig *testutil.FailureConfig + reconcilerScheme *runtime.Scheme + wantErr bool + assertFunc func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) }{ ////---------------------------------------- /// Success @@ -403,6 +404,18 @@ func TestShardReconciler_Reconcile(t *testing.T) { Namespace: "default", }, }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-hba-template", + Namespace: "default", + }, + }, + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-existing-shard-pool-primary-zone1", + Namespace: "default", + }, + }, }, assertFunc: func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) { poolSts := &appsv1.StatefulSet{} @@ -1460,33 +1473,310 @@ func TestShardReconciler_Reconcile(t *testing.T) { existingObjects: []client.Object{ &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-multiorch", + Name: "test-shard-status-multiorch-zone1", Namespace: "default", }, }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-multiorch", + Name: "test-shard-status-multiorch-zone1", Namespace: "default", }, }, &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-pool-primary", + Name: "test-shard-status-pool-primary-zone1", Namespace: "default", }, }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-pool-primary-headless", + Name: "test-shard-status-pool-primary-zone1-headless", Namespace: "default", }, }, }, failureConfig: &testutil.FailureConfig{ // Fail Pool StatefulSet Get after successful reconciliation calls - // Get calls: 1=Shard, 2=MultiOrchDeploy, 3=MultiOrchSvc, 4=PoolSts, 5=PoolSvc, 6=PoolSts(status) - OnGet: testutil.FailKeyAfterNCalls(5, testutil.ErrNetworkTimeout), + // Get calls: 1=Shard, 2=PgHbaCM, 3=MultiOrchDeploy, 4=MultiOrchSvc, 5=PoolBackupPVC, 6=PoolSts, 7=PoolSvc, 8=PoolSts(status) + OnGet: testutil.FailKeyAfterNCalls(7, testutil.ErrNetworkTimeout), + }, + wantErr: true, + }, + "error on build PgHba ConfigMap (empty scheme)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + reconcilerScheme: runtime.NewScheme(), // Empty scheme + wantErr: true, + }, + "error on Get PgHba ConfigMap (network error)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba-get-err", + Namespace: "default", + }, + }, + failureConfig: &testutil.FailureConfig{ + OnGet: func(key client.ObjectKey) error { + if key.Name == "pg-hba-template" { + return testutil.ErrNetworkTimeout + } + return nil + }, + }, + wantErr: true, + }, + "error on build MultiOrch Deployment (scheme missing Shard)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-mo-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + }, + }, + reconcilerScheme: func() *runtime.Scheme { + s := runtime.NewScheme() + // Register types but NOT Shard to fail SetControllerReference + _ = corev1.AddToScheme(s) + _ = appsv1.AddToScheme(s) + return s + }(), + wantErr: true, + }, + "error on build Pool BackupPVC (scheme missing Shard)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-pvc-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + reconcilerScheme: func() *runtime.Scheme { + s := runtime.NewScheme() + _ = corev1.AddToScheme(s) + _ = appsv1.AddToScheme(s) + // Shard is NOT added + return s + }(), + wantErr: true, + }, + "error on PgHba ConfigMap create": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnCreate: func(obj client.Object) error { + if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Name == "pg-hba-template" { + return testutil.ErrPermissionError + } + return nil + }, + }, + wantErr: true, + }, + "error on PgHba ConfigMap update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + existingObjects: []client.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-hba-template", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Name == "pg-hba-template" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC create": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnCreate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrPermissionError + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{ + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pvc-pool-primary-zone1", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC labels update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc-label-err", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + }, + }, + }, + existingObjects: []client.Object{ + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pvc-label-err-pool-primary-zone1", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-label-err-pool-primary-zone1" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Get Pool BackupPVC (network error)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnGet: func(key client.ObjectKey) error { + if key.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrNetworkTimeout + } + return nil + }, }, wantErr: true, }, @@ -1513,6 +1803,9 @@ func TestShardReconciler_Reconcile(t *testing.T) { Client: fakeClient, Scheme: scheme, } + if tc.reconcilerScheme != nil { + reconciler.Scheme = tc.reconcilerScheme + } // Create the Shard resource if not in existing objects shardInExisting := false diff --git a/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf b/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf new file mode 100644 index 00000000..2701ddbb --- /dev/null +++ b/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf @@ -0,0 +1,36 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Generated by Multigres - Helping you Scale PostgreSQL +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all {{.User}} trust +local all all peer + +# Replication connections - trust from all IPs for testing +# PRODUCTION: Replace 'trust' with 'scram-sha-256' and configure passwords +local replication all trust +host replication all 0.0.0.0/0 trust +host replication all ::0/0 trust + +# IPv4 local connections: +host all all 0.0.0.0/0 trust +host all all ::0/0 trust + +# IPv4 external connections +host all all 0.0.0.0/0 trust +host all all ::0/0 trust + +# IPv6 external connections +host all all ::0/0 trust diff --git a/pkg/resource-handler/controller/storage/pvc.go b/pkg/resource-handler/controller/storage/pvc.go index 84be0773..cc1b113f 100644 --- a/pkg/resource-handler/controller/storage/pvc.go +++ b/pkg/resource-handler/controller/storage/pvc.go @@ -15,19 +15,25 @@ import ( // - name: Name for the volume claim (e.g., "data") // - storageClassName: Optional storage class name (nil uses cluster default) // - storageSize: Size of the volume (e.g., "10Gi") +// - accessModes: List of access modes (e.g., [ReadWriteOnce]). Defaults to ReadWriteOnce if empty. func BuildPVCTemplate( name string, storageClassName *string, storageSize string, + accessModes []corev1.PersistentVolumeAccessMode, ) corev1.PersistentVolumeClaim { + if len(accessModes) == 0 { + accessModes = []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + } + } + pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, + AccessModes: accessModes, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse(storageSize), diff --git a/pkg/resource-handler/controller/storage/pvc_test.go b/pkg/resource-handler/controller/storage/pvc_test.go index 42c3b960..f6918cff 100644 --- a/pkg/resource-handler/controller/storage/pvc_test.go +++ b/pkg/resource-handler/controller/storage/pvc_test.go @@ -17,6 +17,7 @@ func TestBuildPVCTemplate(t *testing.T) { name string storageClassName *string storageSize string + accessModes []corev1.PersistentVolumeAccessMode want corev1.PersistentVolumeClaim }{ "basic case with storage class and size": { @@ -122,11 +123,35 @@ func TestBuildPVCTemplate(t *testing.T) { }, }, }, + "custom access modes": { + name: "backup-data", + storageClassName: &storageClassStandard, + storageSize: "50Gi", + accessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + want: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + StorageClassName: &storageClassStandard, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("50Gi"), + }, + }, + }, + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - got := BuildPVCTemplate(tc.name, tc.storageClassName, tc.storageSize) + got := BuildPVCTemplate(tc.name, tc.storageClassName, tc.storageSize, tc.accessModes) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("BuildPVCTemplate() mismatch (-want +got):\n%s", diff) } diff --git a/pkg/resource-handler/controller/toposerver/statefulset.go b/pkg/resource-handler/controller/toposerver/statefulset.go index 3a43717f..a6263c72 100644 --- a/pkg/resource-handler/controller/toposerver/statefulset.go +++ b/pkg/resource-handler/controller/toposerver/statefulset.go @@ -120,6 +120,7 @@ func buildVolumeClaimTemplates( ) []corev1.PersistentVolumeClaim { var storageClass *string storageSize := DefaultStorageSize + var accessModes []corev1.PersistentVolumeAccessMode if toposerver.Spec.Etcd != nil { if toposerver.Spec.Etcd.Storage.Class != "" { @@ -128,9 +129,10 @@ func buildVolumeClaimTemplates( if toposerver.Spec.Etcd.Storage.Size != "" { storageSize = toposerver.Spec.Etcd.Storage.Size } + accessModes = toposerver.Spec.Etcd.Storage.AccessModes } return []corev1.PersistentVolumeClaim{ - storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize), + storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize, accessModes), } } diff --git a/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml b/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml new file mode 100644 index 00000000..a1dcd12c --- /dev/null +++ b/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml @@ -0,0 +1,24 @@ +# createclustermetadata currently only creates the cell metadata +# on the global etcd cluster. It will later be enhanced to accept +# cell-specific topo. +apiVersion: batch/v1 +kind: Job +metadata: + name: createclustermetadata +spec: + template: + spec: + containers: + - name: multigres + image: ghcr.io/multigres/multigres:main + imagePullPolicy: IfNotPresent + command: ["/multigres/bin/multigres"] + args: + - createclustermetadata + - --global-topo-address=kind-toposerver-sample:2379 + - --global-topo-root=/multigres/global + - --cells=zone-a # This demo is for a single zone cluster. + - --durability-policy=ANY_2 # multiorch does not support single node durability policies yet. + - --backup-location=/backups # This path is mounted on all nodes and pods through hostPath. + restartPolicy: OnFailure + backoffLimit: 3 diff --git a/pkg/resource-handler/e2e-config/kind-cell.yaml b/pkg/resource-handler/e2e-config/kind-cell.yaml index f21897a7..3861de08 100644 --- a/pkg/resource-handler/e2e-config/kind-cell.yaml +++ b/pkg/resource-handler/e2e-config/kind-cell.yaml @@ -4,14 +4,20 @@ metadata: name: kind-cell-sample namespace: default spec: + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # This configuration is compatible with the MVP (Minimum Viable Product) + # Logical name of the cell name: zone-a # Physical zone placement (use either 'zone' or 'region', not both) zone: us-west-1a - # Container image for multigateway - multigatewayImage: ghcr.io/multigres/multigres:main + # Container images for cell components + images: + multigateway: ghcr.io/multigres/multigres:main # MultiGateway deployment - query routing multigateway: @@ -26,7 +32,7 @@ spec: # Reference to the global topology server globalTopoServer: - address: kind-global-topo:2379 + address: kind-toposerver-sample:2379 rootPath: /multigres/global implementation: etcd2 @@ -38,6 +44,8 @@ spec: replicas: 1 storage: size: 1Gi + accessModes: + - ReadWriteOnce resources: requests: cpu: 50m diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 06fb476e..efcb5499 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -4,20 +4,37 @@ metadata: name: kind-shard-sample namespace: default spec: - # Shard identification - databaseName: mydb - tableGroupName: users - shardName: "0" + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # The MVP (Minimum Viable Product) has strict validation requirements: + # - Database must be "postgres" + # - TableGroup must be "default" + # - Shard name must be "0-inf" for default tablegroup (unsharded/range mode) + + databaseName: postgres + tableGroupName: default + shardName: "0-inf" + + # ============================================================================ + # Original E2E Configuration (Commented Out) + # ============================================================================ + # For future e2e testing with custom databases/tablegroups, uncomment below + # and comment out the MVP config above: + # + # databaseName: mydb + # tableGroupName: users + # shardName: "0" # Container images for shard components images: multiorch: ghcr.io/multigres/multigres:main multipooler: ghcr.io/multigres/multigres:main - postgres: postgres:16-alpine + postgres: ghcr.io/multigres/pgctld:main # Reference to the global topology server globalTopoServer: - address: kind-global-topo:2379 + address: kind-toposerver-sample:2379 rootPath: /multigres/global implementation: etcd2 @@ -36,16 +53,18 @@ spec: # Shard pools - different types of PostgreSQL replicas pools: - # Primary replica pool - primary: + # Primary "eligible" pool, with unique name + pool-1: type: readWrite cells: - zone-a - replicasPerCell: 1 + replicasPerCell: 3 # Storage configuration - kind uses local storage provisioner storage: size: 1Gi + accessModes: + - ReadWriteOnce # PostgreSQL container configuration - lower resources for kind postgres: @@ -67,30 +86,31 @@ spec: cpu: 200m memory: 128Mi - # Read replica pool - replica: - type: readOnly - cells: - - zone-a - replicasPerCell: 1 + # Commenting out for isolated testing + # # Read replica pool + # pool-readonly: + # type: readOnly + # cells: + # - zone-a + # replicasPerCell: 3 - storage: - size: 1Gi + # storage: + # size: 1Gi - postgres: - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi + # postgres: + # resources: + # requests: + # cpu: 100m + # memory: 256Mi + # limits: + # cpu: 500m + # memory: 512Mi - multipooler: - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi + # multipooler: + # resources: + # requests: + # cpu: 50m + # memory: 64Mi + # limits: + # cpu: 200m + # memory: 128Mi diff --git a/pkg/resource-handler/e2e-config/kind-toposerver.yaml b/pkg/resource-handler/e2e-config/kind-toposerver.yaml index d937122e..dda76279 100644 --- a/pkg/resource-handler/e2e-config/kind-toposerver.yaml +++ b/pkg/resource-handler/e2e-config/kind-toposerver.yaml @@ -4,6 +4,11 @@ metadata: name: kind-toposerver-sample namespace: default spec: + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # Minimal etcd configuration compatible with MVP requirements + # Etcd configuration for the topology server etcd: # etcd container image @@ -16,6 +21,8 @@ spec: # Storage configuration for etcd data - kind uses local storage provisioner storage: size: 1Gi + accessModes: + - ReadWriteOnce # Resource requirements for etcd containers - lower for kind resources: diff --git a/pkg/resource-handler/go.mod b/pkg/resource-handler/go.mod index 47629df9..b8a7e6af 100644 --- a/pkg/resource-handler/go.mod +++ b/pkg/resource-handler/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/google/go-cmp v0.7.0 - github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b + github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805 github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373 k8s.io/api v0.34.3 k8s.io/apimachinery v0.34.3 diff --git a/pkg/resource-handler/go.sum b/pkg/resource-handler/go.sum index 964f3275..f5773f55 100644 --- a/pkg/resource-handler/go.sum +++ b/pkg/resource-handler/go.sum @@ -90,8 +90,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b h1:twErev/AdyVQvlYa5vbo5wqq+NlexZJmIm0pwEQd5qI= -github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b/go.mod h1:A1bBmTxHr+362dGZ5G6u2S4xsP6enbgdUS/UJUOmKbc= +github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805 h1:XUmBsLf2Lv1AIRkPLN8ipn/QBnOlJ7SGu1RidhovhwY= +github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805/go.mod h1:A1bBmTxHr+362dGZ5G6u2S4xsP6enbgdUS/UJUOmKbc= github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373 h1:B9uGjUsG0rMi+dGt2blEDpr8wbwnE/W1xcpuxZwvOYk= github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373/go.mod h1:+NQa7dSvQqxhBOE9XcE9RWXLvOvNaw0keCc29Y7pjyQ= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= diff --git a/plans/phase-1/makefile-streamlining.md b/plans/phase-1/makefile-streamlining.md index bd08d7c9..7868374e 100644 --- a/plans/phase-1/makefile-streamlining.md +++ b/plans/phase-1/makefile-streamlining.md @@ -314,12 +314,12 @@ kind-deploy: kind-up manifests kustomize kind-load ## Deploy operator to kind cl cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/default | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f - @echo "==> Deployment complete!" - @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator-system" + @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator" .PHONY: kind-redeploy kind-redeploy: kind-load ## Rebuild image, reload to kind, and restart pods @echo "==> Restarting operator pods..." - KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator-system + KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator .PHONY: kind-down kind-down: ## Delete the kind cluster @@ -595,7 +595,7 @@ jobs: # Deploy operator make kind-deploy - KUBECONFIG=$(pwd)/kubeconfig.yaml kubectl get pods -n multigres-operator-system + KUBECONFIG=$(pwd)/kubeconfig.yaml kubectl get pods -n multigres-operator # Make code changes and redeploy make kind-redeploy