From 8da77615bd159bfb715693c7c4d10706a5839f86 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 03:19:35 +0000 Subject: [PATCH 01/58] Correct resource CLI arguments --- .../controller/cell/integration_test.go | 4 --- .../controller/cell/multigateway.go | 1 - .../controller/cell/multigateway_test.go | 5 --- .../controller/shard/containers.go | 20 +++++------ .../controller/shard/containers_test.go | 13 +++---- .../controller/shard/integration_test.go | 34 +++++++------------ 6 files changed, 26 insertions(+), 51 deletions(-) diff --git a/pkg/resource-handler/controller/cell/integration_test.go b/pkg/resource-handler/controller/cell/integration_test.go index b4a16e52..67cefe80 100644 --- a/pkg/resource-handler/controller/cell/integration_test.go +++ b/pkg/resource-handler/controller/cell/integration_test.go @@ -114,7 +114,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Ports: []corev1.ContainerPort{ @@ -201,7 +200,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", }, Ports: []corev1.ContainerPort{ @@ -288,7 +286,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone3", }, Ports: []corev1.ContainerPort{ @@ -392,7 +389,6 @@ func TestCellReconciliation(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone4", }, Ports: []corev1.ContainerPort{ diff --git a/pkg/resource-handler/controller/cell/multigateway.go b/pkg/resource-handler/controller/cell/multigateway.go index d5d0110b..ca918cc7 100644 --- a/pkg/resource-handler/controller/cell/multigateway.go +++ b/pkg/resource-handler/controller/cell/multigateway.go @@ -79,7 +79,6 @@ func BuildMultiGatewayDeployment( "--pg-port", fmt.Sprintf("%d", MultiGatewayPostgresPort), "--topo-global-server-addresses", cell.Spec.GlobalTopoServer.Address, "--topo-global-root", cell.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", cell.Spec.GlobalTopoServer.Implementation, "--cell", cell.Spec.Name, }, Resources: cell.Spec.MultiGateway.Resources, diff --git a/pkg/resource-handler/controller/cell/multigateway_test.go b/pkg/resource-handler/controller/cell/multigateway_test.go index 013361ae..f9eea787 100644 --- a/pkg/resource-handler/controller/cell/multigateway_test.go +++ b/pkg/resource-handler/controller/cell/multigateway_test.go @@ -97,7 +97,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Resources: corev1.ResourceRequirements{}, @@ -200,7 +199,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", }, Resources: corev1.ResourceRequirements{}, @@ -303,7 +301,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone3", }, Resources: corev1.ResourceRequirements{}, @@ -422,7 +419,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone4", }, Resources: corev1.ResourceRequirements{}, @@ -551,7 +547,6 @@ func TestBuildMultiGatewayDeployment(t *testing.T) { "--pg-port", "15432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone5", }, Resources: corev1.ResourceRequirements{ diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index d93f6a60..7df2c3c1 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -78,7 +78,8 @@ func buildMultiPoolerSidecar( // TODO: Add remaining command line arguments: // --pooler-dir, --grpc-socket-file, --log-level, --log-output, --hostname, --service-map - // --pgbackrest-stanza, --connpool-admin-password, --socket-file + // --connpool-admin-password, --socket-file + // Note: --pgbackrest-stanza removed in multigres PR #398 (now hardcoded to "multigres") args := []string{ "multipooler", // Subcommand @@ -86,7 +87,6 @@ func buildMultiPoolerSidecar( "--grpc-port", "15270", "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", shard.Spec.GlobalTopoServer.Implementation, "--cell", cellName, "--database", shard.Spec.DatabaseName, "--table-group", shard.Spec.TableGroupName, @@ -107,18 +107,19 @@ func buildMultiPoolerSidecar( } // buildPgctldInitContainer creates the pgctld init container spec. -// This copies the pgctld binary to a shared volume for use by the postgres container. +// NOTE: As of multigres PR #398, pgctld is no longer in the main multigres image. +// The new approach uses a combined postgres+pgctld image (multigres/pgctld-postgres). +// This init container uses a shell fallback since 'pgctld copy-binary' command was removed. +// TODO: Remove this init container and use multigres/pgctld-postgres image directly for postgres container. func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { image := DefaultMultigresImage - // TODO: Add pgctld image field to Shard spec if needed return corev1.Container{ - Name: "pgctld-init", - Image: image, + Name: "pgctld-init", + Image: image, + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", // Subcommand - "copy-binary", - "--output", "/shared/pgctld", + "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", }, VolumeMounts: []corev1.VolumeMount{ { @@ -146,7 +147,6 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co "--grpc-port", "15370", "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, - "--topo-implementation", shard.Spec.GlobalTopoServer.Implementation, "--cell", cellName, } diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 6f37080a..0a20d6c4 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -153,7 +153,6 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "testdb", "--table-group", "default", @@ -197,7 +196,6 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", "--database", "proddb", "--table-group", "orders", @@ -250,7 +248,6 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "mydb", "--table-group", "default", @@ -296,12 +293,11 @@ func TestBuildPgctldInitContainer(t *testing.T) { Spec: multigresv1alpha1.ShardSpec{}, }, want: corev1.Container{ - Name: "pgctld-init", - Image: DefaultMultigresImage, + Name: "pgctld-init", + Image: DefaultMultigresImage, + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", }, VolumeMounts: []corev1.VolumeMount{ { @@ -350,7 +346,6 @@ func TestBuildMultiOrchContainer(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Ports: buildMultiOrchContainerPorts(), diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 6aa95c5e..d841fdd7 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -122,7 +122,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "us-west-1a", }, Ports: []corev1.ContainerPort{ @@ -180,7 +179,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "us-west-1b", }, Ports: []corev1.ContainerPort{ @@ -234,12 +232,11 @@ func TestShardReconciliation(t *testing.T) { Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Name: "pgctld-init", + Image: "ghcr.io/multigres/multigres:main", + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, @@ -254,7 +251,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "us-west-1a", "--database", "testdb", "--table-group", "default", @@ -393,7 +389,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", }, Ports: []corev1.ContainerPort{ @@ -451,7 +446,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", }, Ports: []corev1.ContainerPort{ @@ -510,12 +504,11 @@ func TestShardReconciliation(t *testing.T) { }, InitContainers: []corev1.Container{ { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Name: "pgctld-init", + Image: "ghcr.io/multigres/multigres:main", + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, @@ -530,7 +523,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone1", "--database", "testdb", "--table-group", "default", @@ -629,12 +621,11 @@ func TestShardReconciliation(t *testing.T) { }, InitContainers: []corev1.Container{ { - Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Name: "pgctld-init", + Image: "ghcr.io/multigres/multigres:main", + Command: []string{"/bin/sh", "-c"}, Args: []string{ - "pgctld", - "copy-binary", - "--output", "/shared/pgctld", + "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, @@ -649,7 +640,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--topo-implementation", "etcd2", "--cell", "zone2", "--database", "testdb", "--table-group", "default", From 260d579d026c9fedd9103af7943e15892bc9ccbe Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 04:00:19 +0000 Subject: [PATCH 02/58] Update pgctld copy to use different image --- .../controller/shard/containers.go | 21 +++++++++++-------- .../controller/shard/containers_test.go | 4 ++-- .../controller/shard/integration_test.go | 12 +++++------ 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 7df2c3c1..5d1a43b4 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -9,10 +9,13 @@ import ( ) const ( - // DefaultMultigresImage is the base image for all Multigres components (multipooler, multiorch, pgctld) + // DefaultMultigresImage is the base image for all Multigres components (multipooler, multiorch) // Different components use different subcommands. DefaultMultigresImage = "ghcr.io/multigres/multigres:main" + // DefaultPgctldImage is the image containing the pgctld binary + DefaultPgctldImage = "ghcr.io/multigres/pgctld:main" + // DefaultPostgresImage is the default PostgreSQL database container image DefaultPostgresImage = "postgres:17" @@ -78,8 +81,7 @@ func buildMultiPoolerSidecar( // TODO: Add remaining command line arguments: // --pooler-dir, --grpc-socket-file, --log-level, --log-output, --hostname, --service-map - // --connpool-admin-password, --socket-file - // Note: --pgbackrest-stanza removed in multigres PR #398 (now hardcoded to "multigres") + // --pgbackrest-stanza, --connpool-admin-password, --socket-file args := []string{ "multipooler", // Subcommand @@ -107,19 +109,20 @@ func buildMultiPoolerSidecar( } // buildPgctldInitContainer creates the pgctld init container spec. -// NOTE: As of multigres PR #398, pgctld is no longer in the main multigres image. -// The new approach uses a combined postgres+pgctld image (multigres/pgctld-postgres). -// This init container uses a shell fallback since 'pgctld copy-binary' command was removed. -// TODO: Remove this init container and use multigres/pgctld-postgres image directly for postgres container. +// This copies the pgctld binary from the dedicated pgctld image to a shared volume +// for use by the postgres container. func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { - image := DefaultMultigresImage + // Use dedicated pgctld image (ghcr.io/multigres/pgctld:main) + // TODO: Add pgctld image field to Shard spec if users need to override + image := DefaultPgctldImage return corev1.Container{ Name: "pgctld-init", Image: image, Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", + // Copy pgctld from /usr/local/bin/pgctld (location in pgctld image) to shared volume + "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ { diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 0a20d6c4..a4988b78 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -294,10 +294,10 @@ func TestBuildPgctldInitContainer(t *testing.T) { }, want: corev1.Container{ Name: "pgctld-init", - Image: DefaultMultigresImage, + Image: DefaultPgctldImage, Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", + "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ { diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index d841fdd7..529e10ec 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -233,10 +233,10 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", + "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, @@ -505,10 +505,10 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", + "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, @@ -622,10 +622,10 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /multigres/bin/pgctld /shared/pgctld 2>/dev/null || echo 'pgctld not found in image, skipping copy'", + "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgctld-bin", MountPath: "/shared"}, From 600c0ec13ecec056bf4a906cd295a733fcbf1fa2 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:02:12 +0000 Subject: [PATCH 03/58] Ensure watch-targets is passed down This currently defaults to "postgres" as per demo setup in the upstream repository. --- pkg/resource-handler/controller/shard/containers.go | 8 +++++++- pkg/resource-handler/controller/shard/containers_test.go | 1 + pkg/resource-handler/controller/shard/integration_test.go | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 5d1a43b4..8e402e3d 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -141,9 +141,14 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } // TODO: Add remaining command line arguments: - // --watch-targets, --log-level, --log-output, --hostname + // --log-level, --log-output, --hostname // --cluster-metadata-refresh-interval, --pooler-health-check-interval, --recovery-cycle-interval + // TODO: Verify correct format for --watch-targets flag. + // Currently using static "postgres" based on demo, but may need to be: + // - Just database name (e.g., "postgres") + // - Full path (e.g., "database/tablegroup/shard") + // - Multiple targets (e.g., "postgres,otherdb") args := []string{ "multiorch", // Subcommand "--http-port", "15300", @@ -151,6 +156,7 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, "--cell", cellName, + "--watch-targets", "postgres", } return corev1.Container{ diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index a4988b78..7ef84a3e 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -347,6 +347,7 @@ func TestBuildMultiOrchContainer(t *testing.T) { "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", + "--watch-targets", "postgres", }, Ports: buildMultiOrchContainerPorts(), Resources: corev1.ResourceRequirements{}, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 529e10ec..5bfb2512 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -123,6 +123,7 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "us-west-1a", + "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -180,6 +181,7 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "us-west-1b", + "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -390,6 +392,7 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", + "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -447,6 +450,7 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone2", + "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), From ba328fcf97923fc2ba39d9d155e59e7d48d0e495 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:02:39 +0000 Subject: [PATCH 04/58] Correct image handling for multigateway --- pkg/resource-handler/e2e-config/kind-cell.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-cell.yaml b/pkg/resource-handler/e2e-config/kind-cell.yaml index f21897a7..a7da3752 100644 --- a/pkg/resource-handler/e2e-config/kind-cell.yaml +++ b/pkg/resource-handler/e2e-config/kind-cell.yaml @@ -10,8 +10,9 @@ spec: # Physical zone placement (use either 'zone' or 'region', not both) zone: us-west-1a - # Container image for multigateway - multigatewayImage: ghcr.io/multigres/multigres:main + # Container images for cell components + images: + multigateway: ghcr.io/multigres/multigres:main # MultiGateway deployment - query routing multigateway: From ca4c7e5af83a2ddc4195e3a8392eec1916a07a9e Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:06:31 +0000 Subject: [PATCH 05/58] Correct CRD installation --- config/crd/kustomization.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 13137a10..c29bc63f 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,11 +2,15 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: +# Parent CRDs +- bases/multigres.com_multigresclusters.yaml +- bases/multigres.com_tablegroups.yaml +# Template CRDs +- bases/multigres.com_celltemplates.yaml +- bases/multigres.com_coretemplates.yaml +- bases/multigres.com_shardtemplates.yaml +# Child CRDs - bases/multigres.com_cells.yaml -- bases/multigres.com_etcds.yaml -- bases/multigres.com_multigateways.yaml -- bases/multigres.com_multiorches.yaml -- bases/multigres.com_multipoolers.yaml - bases/multigres.com_shards.yaml - bases/multigres.com_toposervers.yaml # +kubebuilder:scaffold:crdkustomizeresource From b6f0b71f28b3dd5747f000d89b6f5655c4b535b7 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:13:27 +0000 Subject: [PATCH 06/58] Correct caching logic for coverage CI --- .github/workflows/_reusable-test-coverage.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_reusable-test-coverage.yaml b/.github/workflows/_reusable-test-coverage.yaml index 7b1301e2..76a5aeab 100644 --- a/.github/workflows/_reusable-test-coverage.yaml +++ b/.github/workflows/_reusable-test-coverage.yaml @@ -66,8 +66,9 @@ jobs: ~/.cache/go-build ~/go/pkg/mod bin - key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum', 'Makefile') }} + key: ${{ runner.os }}-go-coverage-${{ hashFiles('**/go.sum', 'Makefile') }} restore-keys: | + ${{ runner.os }}-go-coverage- ${{ runner.os }}-go-build- - name: Build @@ -246,11 +247,12 @@ jobs: - name: Save cache if: always() && github.event_name != 'pull_request' - uses: actions/cache/save@v3 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - key: ${{ steps.go-cache.outputs.cache-primary-key }} + key: ${{ runner.os }}-go-coverage-${{ hashFiles('**/go.sum', 'Makefile') }} # Any location that we generate the test coverage report in path: | ~/.cache/coverage.txt ~/.cache/go-build - ~/go/pkg/mod \ No newline at end of file + ~/go/pkg/mod + bin \ No newline at end of file From 9d6dd13b9ae017d32a22b2a0d17f7915757e40ff Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:31:47 +0000 Subject: [PATCH 07/58] Use server side apply for large CRD def --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3d7968a7..c5ba4cde 100644 --- a/Makefile +++ b/Makefile @@ -420,10 +420,10 @@ kind-load: container ## Build and load image into kind cluster .PHONY: kind-deploy kind-deploy: kind-up manifests kustomize kind-load ## Deploy operator to kind cluster @echo "==> Installing CRDs..." - KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/crd | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f - + KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/crd | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply --server-side -f - @echo "==> Deploying operator..." cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/default | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f - + KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/default | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply --server-side -f - @echo "==> Deployment complete!" @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator-system" From 719467470a2bbb6f6353eeb11c89c3498b8aa241 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 06:53:36 +0000 Subject: [PATCH 08/58] Update database and tablegroup to be default --- pkg/resource-handler/e2e-config/kind-shard.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 06fb476e..f76f2798 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -5,8 +5,9 @@ metadata: namespace: default spec: # Shard identification - databaseName: mydb - tableGroupName: users + # Note: v1alpha1 API enforces "postgres" database and "default" tablegroup + databaseName: postgres + tableGroupName: default shardName: "0" # Container images for shard components From cd64f4891dbd16b6ec9a33260bae26597b101d4f Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 15:44:58 +0000 Subject: [PATCH 09/58] Add note about MVP implementation --- .../e2e-config/kind-cell.yaml | 5 +++++ .../e2e-config/kind-shard.yaml | 22 ++++++++++++++++--- .../e2e-config/kind-toposerver.yaml | 5 +++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-cell.yaml b/pkg/resource-handler/e2e-config/kind-cell.yaml index a7da3752..063033f2 100644 --- a/pkg/resource-handler/e2e-config/kind-cell.yaml +++ b/pkg/resource-handler/e2e-config/kind-cell.yaml @@ -4,6 +4,11 @@ metadata: name: kind-cell-sample namespace: default spec: + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # This configuration is compatible with the MVP (Minimum Viable Product) + # Logical name of the cell name: zone-a diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index f76f2798..241be3aa 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -4,11 +4,27 @@ metadata: name: kind-shard-sample namespace: default spec: - # Shard identification - # Note: v1alpha1 API enforces "postgres" database and "default" tablegroup + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # The MVP (Minimum Viable Product) has strict validation requirements: + # - Database must be "postgres" + # - TableGroup must be "default" + # - Shard name must be "0-inf" for default tablegroup (unsharded/range mode) + databaseName: postgres tableGroupName: default - shardName: "0" + shardName: "0-inf" + + # ============================================================================ + # Original E2E Configuration (Commented Out) + # ============================================================================ + # For future e2e testing with custom databases/tablegroups, uncomment below + # and comment out the MVP config above: + # + # databaseName: mydb + # tableGroupName: users + # shardName: "0" # Container images for shard components images: diff --git a/pkg/resource-handler/e2e-config/kind-toposerver.yaml b/pkg/resource-handler/e2e-config/kind-toposerver.yaml index d937122e..838cdc57 100644 --- a/pkg/resource-handler/e2e-config/kind-toposerver.yaml +++ b/pkg/resource-handler/e2e-config/kind-toposerver.yaml @@ -4,6 +4,11 @@ metadata: name: kind-toposerver-sample namespace: default spec: + # ============================================================================ + # MVP Configuration (Currently Active) + # ============================================================================ + # Minimal etcd configuration compatible with MVP requirements + # Etcd configuration for the topology server etcd: # etcd container image From 2c8af2b54e23f59fadeba59c32455588522ecc5e Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 15:45:11 +0000 Subject: [PATCH 10/58] Correct topo server reference --- pkg/resource-handler/e2e-config/kind-cell.yaml | 2 +- pkg/resource-handler/e2e-config/kind-shard.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-cell.yaml b/pkg/resource-handler/e2e-config/kind-cell.yaml index 063033f2..377be80a 100644 --- a/pkg/resource-handler/e2e-config/kind-cell.yaml +++ b/pkg/resource-handler/e2e-config/kind-cell.yaml @@ -32,7 +32,7 @@ spec: # Reference to the global topology server globalTopoServer: - address: kind-global-topo:2379 + address: kind-toposerver-sample:2379 rootPath: /multigres/global implementation: etcd2 diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 241be3aa..50b12c5c 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -34,7 +34,7 @@ spec: # Reference to the global topology server globalTopoServer: - address: kind-global-topo:2379 + address: kind-toposerver-sample:2379 rootPath: /multigres/global implementation: etcd2 From a44f32c978d31aeee1b452e1d04326735762462f Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 15:48:16 +0000 Subject: [PATCH 11/58] Add note about MVP demo specific env --- pkg/resource-handler/controller/shard/containers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 8e402e3d..2129208f 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -50,6 +50,13 @@ func buildPostgresContainer( Name: "postgres", Image: image, Resources: pool.Postgres.Resources, + Env: []corev1.EnvVar{ + // NOTE: This is for MVP demo setup. + { + Name: "POSTGRES_HOST_AUTH_METHOD", + Value: "trust", + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, From d197ae0656df45838e7daac85bd03bab51193d2f Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 15:49:18 +0000 Subject: [PATCH 12/58] Add e2e helper script for data-handler logic --- pkg/resource-handler/e2e-config/README.md | 54 +++++++++++++++ .../e2e-config/register-topology.sh | 66 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 pkg/resource-handler/e2e-config/README.md create mode 100755 pkg/resource-handler/e2e-config/register-topology.sh diff --git a/pkg/resource-handler/e2e-config/README.md b/pkg/resource-handler/e2e-config/README.md new file mode 100644 index 00000000..36688f0b --- /dev/null +++ b/pkg/resource-handler/e2e-config/README.md @@ -0,0 +1,54 @@ +# E2E Configuration Files + +This directory contains sample CRDs for end-to-end testing and local development. + +## Files + +- **kind-cell.yaml**: Cell resource definition +- **kind-shard.yaml**: Shard resource definition +- **kind-toposerver.yaml**: TopoServer resource definition +- **register-topology.sh**: Script to register topology metadata + +## MVP Configuration + +The current configurations are set up for the MVP (Minimum Viable Product) with strict requirements: + +- Database must be `"postgres"` +- TableGroup must be `"default"` +- Shard name must be `"0-inf"` (unsharded range mode) + +Original e2e configurations with custom database/tablegroup names are commented out in the YAML files. + +## Topology Registration (Required Manual Step) + +**Important**: The `resource-handler` operator only manages Kubernetes resources (Deployments, Services, StatefulSets). It does NOT register cells and databases in the topology server (etcd). This will be implemented in the `data-handler` module. + +### Manual Registration for Testing + +After deploying the resources, you must manually register topology metadata: + +```bash +./register-topology.sh zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml +``` + +**All 5 arguments are required**: +1. Cells (comma-separated for multiple: `zone-a,zone-b`) +2. Database name +3. TopoServer service name +4. Namespace +5. Kubeconfig path + +This script uses the `multigres createclustermetadata` command to properly encode topology data as Protocol Buffers. + +### Without Registration + +If you skip this step, multipooler will crash with: +``` +Error: failed to get existing multipooler: Code: UNAVAILABLE +unable to get connection for cell "zone-a" +``` + +## Architecture + +- **resource-handler**: Manages Kubernetes resources (Deployments, Services, StatefulSets) ✅ +- **data-handler**: Will manage topology/data plane operations using multigres CLI/library ⏳ diff --git a/pkg/resource-handler/e2e-config/register-topology.sh b/pkg/resource-handler/e2e-config/register-topology.sh new file mode 100755 index 00000000..c2d87b1a --- /dev/null +++ b/pkg/resource-handler/e2e-config/register-topology.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Helper script to register cells and database in the topology server +# Uses the multigres CLI to properly encode topology data as protobuf. +# +# Usage: ./register-topology.sh +# Example: ./register-topology.sh zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml + +set -euo pipefail + +if [ "$#" -ne 5 ]; then + echo "Error: Requires exactly 5 arguments" + echo "Usage: $0 " + echo "Example: $0 zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml" + echo "" + echo "Note: For multiple cells, use comma-separated: zone-a,zone-b" + exit 1 +fi + +CELLS="$1" +DATABASE="$2" +TOPO_SERVICE="$3" +NAMESPACE="$4" +KUBE_CONFIG="$5" + +GLOBAL_ROOT="/multigres/global" +TOPO_ADDRESS="${TOPO_SERVICE}:2379" + +echo "Registering topology metadata..." +echo " Cells: ${CELLS}" +echo " Database: ${DATABASE}" +echo " Topo Service: ${TOPO_ADDRESS}" +echo " Namespace: ${NAMESPACE}" +echo " Kubeconfig: ${KUBE_CONFIG}" +echo "" + +# Find a pod with the multigres binary to run the command +# Try multiorch pod first, fallback to any pod with multigres image +EXEC_POD=$(kubectl --kubeconfig="$KUBE_CONFIG" get pods -n "$NAMESPACE" \ + -l app.kubernetes.io/component=multiorch \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + +if [ -z "$EXEC_POD" ]; then + echo "Error: No multiorch pod found. Make sure Shard resources are deployed first." + exit 1 +fi + +echo "Using pod '${EXEC_POD}' to run multigres CLI..." +echo "" + +# Run createclustermetadata command +kubectl --kubeconfig="$KUBE_CONFIG" exec -n "$NAMESPACE" "$EXEC_POD" -- \ + /multigres/bin/multigres createclustermetadata \ + --global-topo-address="$TOPO_ADDRESS" \ + --global-topo-root="$GLOBAL_ROOT" \ + --cells="$CELLS" \ + --backup-location=/backup \ + --durability-policy=none + +echo "" +echo "✓ Topology registered successfully" +echo "" +echo "Verify cell registration:" +echo " kubectl --kubeconfig=$KUBE_CONFIG exec -n $NAMESPACE ${EXEC_POD} -- /multigres/bin/multigres getcellnames --global-topo-address=$TOPO_ADDRESS --global-topo-root=$GLOBAL_ROOT" +echo "" +echo "Verify database registration:" +echo " kubectl --kubeconfig=$KUBE_CONFIG exec -n $NAMESPACE ${EXEC_POD} -- /multigres/bin/multigres getdatabasenames --global-topo-address=$TOPO_ADDRESS --global-topo-root=$GLOBAL_ROOT" From 7cb98300e4ba03192d4886a7856ec4264777a635 Mon Sep 17 00:00:00 2001 From: Ryota Date: Mon, 5 Jan 2026 15:49:46 +0000 Subject: [PATCH 13/58] Correct target namespace --- Makefile | 4 ++-- plans/phase-1/makefile-streamlining.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 261f3c2c..e601c898 100644 --- a/Makefile +++ b/Makefile @@ -425,12 +425,12 @@ kind-deploy: kind-up manifests kustomize kind-load ## Deploy operator to kind cl cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/default | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply --server-side -f - @echo "==> Deployment complete!" - @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator-system" + @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator" .PHONY: kind-redeploy kind-redeploy: kind-load ## Rebuild image, reload to kind, and restart pods @echo "==> Restarting operator pods..." - KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator-system + KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator .PHONY: kind-down kind-down: ## Delete the kind cluster diff --git a/plans/phase-1/makefile-streamlining.md b/plans/phase-1/makefile-streamlining.md index bd08d7c9..7868374e 100644 --- a/plans/phase-1/makefile-streamlining.md +++ b/plans/phase-1/makefile-streamlining.md @@ -314,12 +314,12 @@ kind-deploy: kind-up manifests kustomize kind-load ## Deploy operator to kind cl cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) KUBECONFIG=$(KIND_KUBECONFIG) $(KUSTOMIZE) build config/default | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f - @echo "==> Deployment complete!" - @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator-system" + @echo "Check status: KUBECONFIG=$(KIND_KUBECONFIG) kubectl get pods -n multigres-operator" .PHONY: kind-redeploy kind-redeploy: kind-load ## Rebuild image, reload to kind, and restart pods @echo "==> Restarting operator pods..." - KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator-system + KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) rollout restart deployment -n multigres-operator .PHONY: kind-down kind-down: ## Delete the kind cluster @@ -595,7 +595,7 @@ jobs: # Deploy operator make kind-deploy - KUBECONFIG=$(pwd)/kubeconfig.yaml kubectl get pods -n multigres-operator-system + KUBECONFIG=$(pwd)/kubeconfig.yaml kubectl get pods -n multigres-operator # Make code changes and redeploy make kind-redeploy From ba775cdba314f705cf615218f84014d950ae7f11 Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 10:33:13 +0000 Subject: [PATCH 14/58] Add temp env variable POSTGRES_HOST_AUTH_METHOD --- .../controller/shard/containers_test.go | 18 ++++++++++++++++++ .../controller/shard/integration_test.go | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 7ef84a3e..42a1aab8 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -26,6 +26,12 @@ func TestBuildPostgresContainer(t *testing.T) { Name: "postgres", Image: DefaultPostgresImage, Resources: corev1.ResourceRequirements{}, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_HOST_AUTH_METHOD", + Value: "trust", + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -51,6 +57,12 @@ func TestBuildPostgresContainer(t *testing.T) { Name: "postgres", Image: "postgres:16", Resources: corev1.ResourceRequirements{}, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_HOST_AUTH_METHOD", + Value: "trust", + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -94,6 +106,12 @@ func TestBuildPostgresContainer(t *testing.T) { corev1.ResourceMemory: resource.MustParse("4Gi"), }, }, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_HOST_AUTH_METHOD", + Value: "trust", + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 5bfb2512..989440f6 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -269,6 +269,9 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", + Env: []corev1.EnvVar{ + {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, @@ -547,6 +550,9 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", + Env: []corev1.EnvVar{ + {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, @@ -664,6 +670,9 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", + Env: []corev1.EnvVar{ + {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, From a56ec953d1994c8130f5d2bfbaae814eb422c31a Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 10:34:08 +0000 Subject: [PATCH 15/58] Update pgctld image reference --- pkg/resource-handler/controller/shard/containers.go | 4 ++-- pkg/resource-handler/controller/shard/integration_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 2129208f..a8d7e350 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -14,7 +14,7 @@ const ( DefaultMultigresImage = "ghcr.io/multigres/multigres:main" // DefaultPgctldImage is the image containing the pgctld binary - DefaultPgctldImage = "ghcr.io/multigres/pgctld:main" + DefaultPgctldImage = "ghcr.io/multigres/multigres/pgctld:main" // DefaultPostgresImage is the default PostgreSQL database container image DefaultPostgresImage = "postgres:17" @@ -119,7 +119,7 @@ func buildMultiPoolerSidecar( // This copies the pgctld binary from the dedicated pgctld image to a shared volume // for use by the postgres container. func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { - // Use dedicated pgctld image (ghcr.io/multigres/pgctld:main) + // Use dedicated pgctld image (ghcr.io/multigres/multigres/pgctld:main) // TODO: Add pgctld image field to Shard spec if users need to override image := DefaultPgctldImage diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 989440f6..4ea7696e 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -235,7 +235,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", + Image: "ghcr.io/multigres/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", @@ -512,7 +512,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", + Image: "ghcr.io/multigres/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", @@ -632,7 +632,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", + Image: "ghcr.io/multigres/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", From 1cfeb4fb4ecbd2967d355343024969dcbb57a61a Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 11:52:43 +0000 Subject: [PATCH 16/58] Rename pools in sample --- pkg/resource-handler/e2e-config/kind-shard.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 50b12c5c..7e7310e3 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -53,8 +53,8 @@ spec: # Shard pools - different types of PostgreSQL replicas pools: - # Primary replica pool - primary: + # Primary "eligible" pool, with unique name + pool-1: type: readWrite cells: - zone-a @@ -85,7 +85,7 @@ spec: memory: 128Mi # Read replica pool - replica: + pool-readonly: type: readOnly cells: - zone-a From 9c913d71822ebb9b78620fda1bdb948e43e3f80c Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 12:36:38 +0000 Subject: [PATCH 17/58] Update replica count for pools --- pkg/resource-handler/e2e-config/kind-shard.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 7e7310e3..eb734046 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -58,7 +58,7 @@ spec: type: readWrite cells: - zone-a - replicasPerCell: 1 + replicasPerCell: 3 # Storage configuration - kind uses local storage provisioner storage: @@ -89,7 +89,7 @@ spec: type: readOnly cells: - zone-a - replicasPerCell: 1 + replicasPerCell: 3 storage: size: 1Gi From 269070cd17194d01e633e93d757c0c5165ff360b Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 12:37:05 +0000 Subject: [PATCH 18/58] Add job setup for topo registration --- .../hack/kind-job-topo-registration.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml diff --git a/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml b/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml new file mode 100644 index 00000000..a1dcd12c --- /dev/null +++ b/pkg/resource-handler/e2e-config/hack/kind-job-topo-registration.yaml @@ -0,0 +1,24 @@ +# createclustermetadata currently only creates the cell metadata +# on the global etcd cluster. It will later be enhanced to accept +# cell-specific topo. +apiVersion: batch/v1 +kind: Job +metadata: + name: createclustermetadata +spec: + template: + spec: + containers: + - name: multigres + image: ghcr.io/multigres/multigres:main + imagePullPolicy: IfNotPresent + command: ["/multigres/bin/multigres"] + args: + - createclustermetadata + - --global-topo-address=kind-toposerver-sample:2379 + - --global-topo-root=/multigres/global + - --cells=zone-a # This demo is for a single zone cluster. + - --durability-policy=ANY_2 # multiorch does not support single node durability policies yet. + - --backup-location=/backups # This path is mounted on all nodes and pods through hostPath. + restartPolicy: OnFailure + backoffLimit: 3 From a0608d607b6b7171d3ed1c3a02ca2c9b932fa890 Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 19:31:54 +0000 Subject: [PATCH 19/58] Ensure to use postgres 17 The version difference here actually has more implications than just the version of Postgres itself. The user and group setting in 17 uses 999 user ID, and that's the assumption set for the security context setup. --- pkg/resource-handler/e2e-config/kind-shard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index eb734046..1ea4921f 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -30,7 +30,7 @@ spec: images: multiorch: ghcr.io/multigres/multigres:main multipooler: ghcr.io/multigres/multigres:main - postgres: postgres:16-alpine + postgres: postgres:17 # Reference to the global topology server globalTopoServer: From 0e652cbb404d01325464cf568b59cd1cb3af091a Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 19:39:54 +0000 Subject: [PATCH 20/58] Add security context for non-root requirement --- .../controller/shard/containers.go | 11 ++++++ .../controller/shard/containers_test.go | 16 +++++++++ .../controller/shard/integration_test.go | 34 +++++++++++++++++++ .../controller/shard/pool_statefulset.go | 7 +++- .../controller/shard/pool_statefulset_test.go | 12 +++++++ 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index a8d7e350..6486995f 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -4,6 +4,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" ) @@ -57,6 +58,11 @@ func buildPostgresContainer( Value: "trust", }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), // postgres user in postgres:17 image + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -112,6 +118,11 @@ func buildMultiPoolerSidecar( Ports: buildMultiPoolerContainerPorts(), Resources: pool.Multipooler.Resources, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), // Run as postgres user to access pg_data + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, } } diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 42a1aab8..0ab42098 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" ) @@ -32,6 +33,11 @@ func TestBuildPostgresContainer(t *testing.T) { Value: "trust", }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -63,6 +69,11 @@ func TestBuildPostgresContainer(t *testing.T) { Value: "trust", }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -112,6 +123,11 @@ func TestBuildPostgresContainer(t *testing.T) { Value: "trust", }, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 4ea7696e..13a86815 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -263,6 +263,11 @@ func TestShardReconciliation(t *testing.T) { }, Ports: multipoolerPorts(t), RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, }, }, Containers: []corev1.Container{ @@ -271,6 +276,10 @@ func TestShardReconciliation(t *testing.T) { Image: "postgres:17", Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, @@ -501,6 +510,9 @@ func TestShardReconciliation(t *testing.T) { Labels: shardLabels(t, "multi-cell-shard-pool-primary-zone1", "shard-pool", "zone1"), }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, Volumes: []corev1.Volume{ { Name: "pgctld-bin", @@ -544,6 +556,11 @@ func TestShardReconciliation(t *testing.T) { tcpPort(t, "postgres", 5432), }, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, }, }, Containers: []corev1.Container{ @@ -552,6 +569,10 @@ func TestShardReconciliation(t *testing.T) { Image: "postgres:17", Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, @@ -621,6 +642,9 @@ func TestShardReconciliation(t *testing.T) { Labels: shardLabels(t, "multi-cell-shard-pool-primary-zone2", "shard-pool", "zone2"), }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, Volumes: []corev1.Volume{ { Name: "pgctld-bin", @@ -664,6 +688,11 @@ func TestShardReconciliation(t *testing.T) { tcpPort(t, "postgres", 5432), }, RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, }, }, Containers: []corev1.Container{ @@ -673,6 +702,11 @@ func TestShardReconciliation(t *testing.T) { Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 5fcc7b94..4d0d15e1 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" @@ -67,7 +68,11 @@ func BuildPoolStatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ - // Init containers: pgctld copies binary, multipooler is a native sidecar + // Set fsGroup so PVC volumes are writable by postgres user + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image + }, + // Init containers: copy pgctld binary, multipooler is a native sidecar InitContainers: []corev1.Container{ buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 5322f756..d8143c10 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -105,6 +105,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{ @@ -242,6 +245,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{ @@ -378,6 +384,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{ @@ -529,6 +538,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(int64(999)), + }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{ From 6b1aaa1978ec377f34e3a6268a655bb7dd86b3e5 Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 21:45:06 +0000 Subject: [PATCH 21/58] Correct kubeconfig handling --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e601c898..9ae8d8c9 100644 --- a/Makefile +++ b/Makefile @@ -404,12 +404,12 @@ kind-up: ## Create a kind cluster for local development } @if $(KIND) get clusters | grep -q "^$(KIND_CLUSTER)$$"; then \ echo "Kind cluster '$(KIND_CLUSTER)' already exists."; \ + echo "==> Exporting kubeconfig to $(KIND_KUBECONFIG)"; \ + $(KIND) get kubeconfig --name $(KIND_CLUSTER) > $(KIND_KUBECONFIG); \ else \ echo "Creating kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER); \ + $(KIND) create cluster --name $(KIND_CLUSTER) --kubeconfig $(KIND_KUBECONFIG); \ fi - @echo "==> Exporting kubeconfig to $(KIND_KUBECONFIG)" - @$(KIND) get kubeconfig --name $(KIND_CLUSTER) > $(KIND_KUBECONFIG) @echo "==> Cluster ready. Use: export KUBECONFIG=$(KIND_KUBECONFIG)" .PHONY: kind-load From a3137cad67be087890927a49d02190d66f803bd1 Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 22:47:47 +0000 Subject: [PATCH 22/58] Add more const for pooler and pgctld --- .../controller/shard/containers.go | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 6486995f..c2e01f83 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -23,14 +23,39 @@ const ( // PgctldVolumeName is the name of the shared volume for pgctld binary PgctldVolumeName = "pgctld-bin" - // PgctldMountPath is the mount path for pgctld binary in postgres container - PgctldMountPath = "/usr/local/bin/pgctld" + // PgctldBinDir is the directory where pgctld binary is mounted + // Subdirectory avoids shadowing postgres binaries in /usr/local/bin + PgctldBinDir = "/usr/local/bin/multigres" + + // PgctldMountPath is the full path to pgctld binary + PgctldMountPath = PgctldBinDir + "/pgctld" // DataVolumeName is the name of the data volume for PostgreSQL DataVolumeName = "pgdata" - // DataMountPath is the mount path for PostgreSQL data - DataMountPath = "/var/lib/postgresql/data" + // DataMountPath is where the PVC is mounted + // Mounted at parent directory because mounting directly at pg_data/ prevents + // initdb from setting directory permissions (non-root can't chmod mount points). + // pgctld creates pg_data/ subdirectory with proper 0700/0750 permissions. + DataMountPath = "/var/lib/pooler" + + // PgDataPath is the actual postgres data directory (PGDATA env var value) + // pgctld expects postgres data at /pg_data + PgDataPath = "/var/lib/pooler/pg_data" + + // PoolerDirVolumeName exists for historical reasons but shares the same PVC as DataVolumeName + // Both postgres and multipooler mount the same PVC to share pgbackrest configs and sockets + PoolerDirVolumeName = "pooler-dir" + + // PoolerDirMountPath must equal DataMountPath because both containers share the PVC + // and pgctld derives postgres data directory as /pg_data + PoolerDirMountPath = "/var/lib/pooler" + + // SocketDirVolumeName is the name of the shared volume for unix sockets + SocketDirVolumeName = "socket-dir" + + // SocketDirMountPath is the mount path for unix sockets (postgres and pgctld communicate here) + SocketDirMountPath = "/var/run/sockets" ) // sidecarRestartPolicy is the restart policy for native sidecar containers @@ -48,8 +73,21 @@ func buildPostgresContainer( } return corev1.Container{ - Name: "postgres", - Image: image, + Name: "postgres", + Image: image, + Command: []string{PgctldMountPath}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + }, Resources: pool.Postgres.Resources, Env: []corev1.EnvVar{ // NOTE: This is for MVP demo setup. @@ -57,6 +95,10 @@ func buildPostgresContainer( Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust", }, + { + Name: "PGDATA", + Value: PgDataPath, + }, }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), // postgres user in postgres:17 image @@ -70,7 +112,7 @@ func buildPostgresContainer( }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, }, }, } @@ -100,6 +142,8 @@ func buildMultiPoolerSidecar( "multipooler", // Subcommand "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", // Unix socket uses trust auth (no password) "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, "--cell", cellName, @@ -123,6 +167,12 @@ func buildMultiPoolerSidecar( RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, // Shares PVC with postgres for pgbackrest configs and sockets + MountPath: PoolerDirMountPath, + }, + }, } } @@ -136,7 +186,7 @@ func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { return corev1.Container{ Name: "pgctld-init", - Image: image, + Image: DefaultPgctldImage, Command: []string{"/bin/sh", "-c"}, Args: []string{ // Copy pgctld from /usr/local/bin/pgctld (location in pgctld image) to shared volume From 4747d6e3be30c1250533446f111b51f7020a711e Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 22:54:25 +0000 Subject: [PATCH 23/58] Correct pgctld setup --- .../controller/shard/containers.go | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index c2e01f83..375856dd 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -119,10 +119,8 @@ func buildPostgresContainer( } // buildMultiPoolerSidecar creates the multipooler sidecar container spec. -// This is implemented as a native sidecar using init container with -// restartPolicy: Always (K8s 1.28+). -// cellName specifies which cell this container is running in. -// If cellName is empty, defaults to the global topology cell. +// Implemented as native sidecar (init container with restartPolicy: Always) because +// multipooler must restart with postgres to maintain connection pool consistency. func buildMultiPoolerSidecar( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -177,19 +175,13 @@ func buildMultiPoolerSidecar( } // buildPgctldInitContainer creates the pgctld init container spec. -// This copies the pgctld binary from the dedicated pgctld image to a shared volume -// for use by the postgres container. +// Copies pgctld binary to shared volume because postgres image doesn't include it. func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { - // Use dedicated pgctld image (ghcr.io/multigres/multigres/pgctld:main) - // TODO: Add pgctld image field to Shard spec if users need to override - image := DefaultPgctldImage - return corev1.Container{ Name: "pgctld-init", Image: DefaultPgctldImage, Command: []string{"/bin/sh", "-c"}, Args: []string{ - // Copy pgctld from /usr/local/bin/pgctld (location in pgctld image) to shared volume "cp /usr/local/bin/pgctld /shared/pgctld", }, VolumeMounts: []corev1.VolumeMount{ @@ -236,6 +228,17 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } } +// buildPoolerDirVolume creates the emptyDir volume for multipooler working directory. +// This provides writable space for multipooler to create pgbackrest config and other files. +func buildPoolerDirVolume() corev1.Volume { + return corev1.Volume{ + Name: PoolerDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + // buildPgctldVolume creates the shared emptyDir volume for pgctld binary. func buildPgctldVolume() corev1.Volume { return corev1.Volume{ From 05100444d455a98f6b6c282c109f873cc348df83 Mon Sep 17 00:00:00 2001 From: Ryota Date: Tue, 6 Jan 2026 22:55:43 +0000 Subject: [PATCH 24/58] Correct comment and test spec --- .../controller/shard/containers.go | 13 ++- .../controller/shard/containers_test.go | 108 ++++++++++++++++-- .../controller/shard/integration_test.go | 83 ++++++++++++-- .../controller/shard/pool_statefulset.go | 7 +- .../controller/shard/pool_statefulset_test.go | 40 ++----- 5 files changed, 188 insertions(+), 63 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 375856dd..2103dd5e 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -62,7 +62,8 @@ const ( var sidecarRestartPolicy = corev1.ContainerRestartPolicyAlways // buildPostgresContainer creates the postgres container spec for a pool. -// This runs pgctld binary (which wraps postgres) and mounts persistent data storage. +// Uses pgctld server instead of running postgres directly because pgctld provides +// lifecycle management (init, start, stop, status) via gRPC for multiorch/multipooler. func buildPostgresContainer( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -101,9 +102,9 @@ func buildPostgresContainer( }, }, SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To(int64(999)), // postgres user in postgres:17 image + RunAsUser: ptr.To(int64(999)), // Must match postgres:17 image UID for file access RunAsGroup: ptr.To(int64(999)), - RunAsNonRoot: ptr.To(true), + RunAsNonRoot: ptr.To(true), // pgctld refuses to run as root }, VolumeMounts: []corev1.VolumeMount{ { @@ -133,8 +134,8 @@ func buildMultiPoolerSidecar( } // TODO: Add remaining command line arguments: - // --pooler-dir, --grpc-socket-file, --log-level, --log-output, --hostname, --service-map - // --pgbackrest-stanza, --connpool-admin-password, --socket-file + // --grpc-socket-file, --log-level, --log-output, --hostname, --service-map + // --pgbackrest-stanza, --connpool-admin-password args := []string{ "multipooler", // Subcommand @@ -161,7 +162,7 @@ func buildMultiPoolerSidecar( Resources: pool.Multipooler.Resources, RestartPolicy: &sidecarRestartPolicy, SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To(int64(999)), // Run as postgres user to access pg_data + RunAsUser: ptr.To(int64(999)), // Must match postgres UID to access pg_data directory RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 0ab42098..80b0ae42 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -24,14 +24,31 @@ func TestBuildPostgresContainer(t *testing.T) { }, poolSpec: multigresv1alpha1.PoolSpec{}, want: corev1.Container{ - Name: "postgres", - Image: DefaultPostgresImage, + Name: "postgres", + Image: DefaultPostgresImage, + Command: []string{PgctldMountPath}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ { Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust", }, + { + Name: "PGDATA", + Value: PgDataPath, + }, }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), @@ -45,7 +62,7 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, }, }, }, @@ -60,14 +77,31 @@ func TestBuildPostgresContainer(t *testing.T) { }, poolSpec: multigresv1alpha1.PoolSpec{}, want: corev1.Container{ - Name: "postgres", - Image: "postgres:16", + Name: "postgres", + Image: "postgres:16", + Command: []string{PgctldMountPath}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ { Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust", }, + { + Name: "PGDATA", + Value: PgDataPath, + }, }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), @@ -81,7 +115,7 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, }, }, }, @@ -105,8 +139,21 @@ func TestBuildPostgresContainer(t *testing.T) { }, }, want: corev1.Container{ - Name: "postgres", - Image: DefaultPostgresImage, + Name: "postgres", + Image: DefaultPostgresImage, + Command: []string{PgctldMountPath}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), @@ -122,6 +169,10 @@ func TestBuildPostgresContainer(t *testing.T) { Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust", }, + { + Name: "PGDATA", + Value: PgDataPath, + }, }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), @@ -135,7 +186,7 @@ func TestBuildPostgresContainer(t *testing.T) { }, { Name: PgctldVolumeName, - MountPath: PgctldMountPath, + MountPath: PgctldBinDir, }, }, }, @@ -185,6 +236,8 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", @@ -198,6 +251,17 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Ports: buildMultiPoolerContainerPorts(), Resources: corev1.ResourceRequirements{}, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + }, }, }, "custom multipooler image": { @@ -228,6 +292,8 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone2", @@ -241,6 +307,17 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Ports: buildMultiPoolerContainerPorts(), Resources: corev1.ResourceRequirements{}, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + }, }, }, "with resource requirements": { @@ -280,6 +357,8 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", PoolerDirMountPath, + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", @@ -302,6 +381,17 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { }, }, RestartPolicy: &sidecarRestartPolicy, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: PoolerDirMountPath, + }, + }, }, }, } diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 13a86815..6d8fbaf1 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -251,6 +251,8 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "us-west-1a", @@ -268,22 +270,40 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + }, Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, + }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, }, }, }, @@ -540,6 +560,8 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", @@ -561,22 +583,40 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + }, Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, + }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, }, }, }, @@ -672,6 +712,8 @@ func TestShardReconciliation(t *testing.T) { "multipooler", "--http-port", "15200", "--grpc-port", "15270", + "--pooler-dir", "/var/lib/pooler", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone2", @@ -693,14 +735,31 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + }, }, }, Containers: []corev1.Container{ { - Name: "postgres", - Image: "postgres:17", + Name: "postgres", + Image: "postgres:17", + Command: []string{"/usr/local/bin/multigres/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + }, Env: []corev1.EnvVar{ {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, + {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, }, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To(int64(999)), @@ -708,8 +767,8 @@ func TestShardReconciliation(t *testing.T) { RunAsNonRoot: ptr.To(true), }, VolumeMounts: []corev1.VolumeMount{ - {Name: "pgdata", MountPath: "/var/lib/postgresql/data"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/pgctld"}, + {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, }, }, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 4d0d15e1..4673ef28 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -68,22 +68,21 @@ func BuildPoolStatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ - // Set fsGroup so PVC volumes are writable by postgres user + // FSGroup ensures PVC is writable by postgres user (required for initdb) SecurityContext: &corev1.PodSecurityContext{ FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image }, - // Init containers: copy pgctld binary, multipooler is a native sidecar InitContainers: []corev1.Container{ buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, - // Postgres is the main container (runs pgctld binary) Containers: []corev1.Container{ buildPostgresContainer(shard, poolSpec), }, - // Shared volume for pgctld binary Volumes: []corev1.Volume{ buildPgctldVolume(), + // Single PVC shared by both postgres and multipooler because both need + // access to pgbackrest configs, sockets, and postgres data directory }, Affinity: poolSpec.Affinity, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index d8143c10..e1d2479f 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -110,10 +110,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ @@ -131,10 +128,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { Containers: []corev1.Container{ buildPostgresContainer( &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }, multigresv1alpha1.PoolSpec{}, ), @@ -250,10 +244,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ @@ -273,10 +264,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { Containers: []corev1.Container{ buildPostgresContainer( &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }, multigresv1alpha1.PoolSpec{}, ), @@ -389,10 +377,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ @@ -410,10 +395,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { Containers: []corev1.Container{ buildPostgresContainer( &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }, multigresv1alpha1.PoolSpec{}, ), @@ -543,10 +525,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, InitContainers: []corev1.Container{ buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ @@ -564,10 +543,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { Containers: []corev1.Container{ buildPostgresContainer( &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{ - DatabaseName: "testdb", - TableGroupName: "default", - }, + Spec: multigresv1alpha1.ShardSpec{}, }, multigresv1alpha1.PoolSpec{}, ), From e74ed1b43cd6ca92061224e7f639e48b055c687f Mon Sep 17 00:00:00 2001 From: Ryota Date: Wed, 7 Jan 2026 01:51:50 +0000 Subject: [PATCH 25/58] Remove unused code --- .../controller/shard/containers.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 2103dd5e..c503d244 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -43,10 +43,6 @@ const ( // pgctld expects postgres data at /pg_data PgDataPath = "/var/lib/pooler/pg_data" - // PoolerDirVolumeName exists for historical reasons but shares the same PVC as DataVolumeName - // Both postgres and multipooler mount the same PVC to share pgbackrest configs and sockets - PoolerDirVolumeName = "pooler-dir" - // PoolerDirMountPath must equal DataMountPath because both containers share the PVC // and pgctld derives postgres data directory as /pg_data PoolerDirMountPath = "/var/lib/pooler" @@ -229,17 +225,6 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } } -// buildPoolerDirVolume creates the emptyDir volume for multipooler working directory. -// This provides writable space for multipooler to create pgbackrest config and other files. -func buildPoolerDirVolume() corev1.Volume { - return corev1.Volume{ - Name: PoolerDirVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } -} - // buildPgctldVolume creates the shared emptyDir volume for pgctld binary. func buildPgctldVolume() corev1.Volume { return corev1.Volume{ From 40dc20ddbf59c2d79c79c66f823a244d71a6fdf2 Mon Sep 17 00:00:00 2001 From: Ryota Date: Wed, 7 Jan 2026 16:52:23 +0000 Subject: [PATCH 26/58] Correct service-map and service-id handling --- .../controller/shard/containers.go | 25 ++++++------ .../controller/shard/containers_test.go | 39 +++++++++++++++++-- .../controller/shard/integration_test.go | 9 +++-- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index c503d244..9d8830ec 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -1,8 +1,6 @@ package shard import ( - "fmt" - corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -130,7 +128,7 @@ func buildMultiPoolerSidecar( } // TODO: Add remaining command line arguments: - // --grpc-socket-file, --log-level, --log-output, --hostname, --service-map + // --grpc-socket-file, --log-level, --log-output, --hostname // --pgbackrest-stanza, --connpool-admin-password args := []string{ @@ -139,13 +137,14 @@ func buildMultiPoolerSidecar( "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", // Unix socket uses trust auth (no password) + "--service-map", "grpc-pooler", // Only enable grpc-pooler service (disables auto-restore service) "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, "--cell", cellName, "--database", shard.Spec.DatabaseName, "--table-group", shard.Spec.TableGroupName, "--shard", shard.Spec.ShardName, - "--service-id", getPoolServiceID(shard.Name, poolName), + "--service-id", "$(POD_NAME)", // Use pod name as unique service ID "--pgctld-addr", "localhost:15470", "--pg-port", "5432", } @@ -162,6 +161,16 @@ func buildMultiPoolerSidecar( RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, // Shares PVC with postgres for pgbackrest configs and sockets @@ -234,11 +243,3 @@ func buildPgctldVolume() corev1.Volume { }, } } - -// getPoolServiceID generates a unique service ID for a pool. -// This is used in multipooler and pgctld arguments. -func getPoolServiceID(shardName string, poolName string) string { - // TODO: Use proper ID generation (UUID or consistent hash) - // For now, use simple format - return fmt.Sprintf("%s-pool-%s", shardName, poolName) -} diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 80b0ae42..4b914224 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -238,13 +238,14 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "test-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -256,6 +257,16 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -294,13 +305,14 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone2", "--database", "proddb", "--table-group", "orders", "--shard", "1", - "--service-id", "custom-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -312,6 +324,16 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, @@ -359,13 +381,14 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", "--database", "mydb", "--table-group", "default", "--shard", "0", - "--service-id", "resource-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -386,6 +409,16 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: DataVolumeName, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 6d8fbaf1..c1cc57b3 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -253,13 +253,14 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", "/var/lib/pooler", "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "us-west-1a", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "test-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -562,13 +563,14 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", "/var/lib/pooler", "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone1", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "multi-cell-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, @@ -714,13 +716,14 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15270", "--pooler-dir", "/var/lib/pooler", "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", + "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", "--cell", "zone2", "--database", "testdb", "--table-group", "default", "--shard", "0", - "--service-id", "multi-cell-shard-pool-primary", + "--service-id", "$(POD_NAME)", "--pgctld-addr", "localhost:15470", "--pg-port", "5432", }, From 0e2c2ba275d3bdc662f0e4aae76a371ae26d1b8f Mon Sep 17 00:00:00 2001 From: Ryota Date: Wed, 7 Jan 2026 16:58:20 +0000 Subject: [PATCH 27/58] Use simple image name for pgctld --- pkg/resource-handler/controller/shard/containers.go | 2 +- pkg/resource-handler/controller/shard/integration_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 9d8830ec..2e97df40 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -13,7 +13,7 @@ const ( DefaultMultigresImage = "ghcr.io/multigres/multigres:main" // DefaultPgctldImage is the image containing the pgctld binary - DefaultPgctldImage = "ghcr.io/multigres/multigres/pgctld:main" + DefaultPgctldImage = "ghcr.io/multigres/pgctld:main" // DefaultPostgresImage is the default PostgreSQL database container image DefaultPostgresImage = "postgres:17" diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index c1cc57b3..5bf724ff 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -235,7 +235,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres/pgctld:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", @@ -545,7 +545,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres/pgctld:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", @@ -698,7 +698,7 @@ func TestShardReconciliation(t *testing.T) { InitContainers: []corev1.Container{ { Name: "pgctld-init", - Image: "ghcr.io/multigres/multigres/pgctld:main", + Image: "ghcr.io/multigres/pgctld:main", Command: []string{"/bin/sh", "-c"}, Args: []string{ "cp /usr/local/bin/pgctld /shared/pgctld", From fb282c3b9b4bc2f3603dab93530f1115fa49562f Mon Sep 17 00:00:00 2001 From: Ryota Date: Wed, 7 Jan 2026 17:04:25 +0000 Subject: [PATCH 28/58] Temporarily remove the second pool from config --- .../e2e-config/kind-shard.yaml | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 1ea4921f..150a15b9 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -84,30 +84,31 @@ spec: cpu: 200m memory: 128Mi - # Read replica pool - pool-readonly: - type: readOnly - cells: - - zone-a - replicasPerCell: 3 + # Commenting out for isolated testing + # # Read replica pool + # pool-readonly: + # type: readOnly + # cells: + # - zone-a + # replicasPerCell: 3 - storage: - size: 1Gi + # storage: + # size: 1Gi - postgres: - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi + # postgres: + # resources: + # requests: + # cpu: 100m + # memory: 256Mi + # limits: + # cpu: 500m + # memory: 512Mi - multipooler: - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi + # multipooler: + # resources: + # requests: + # cpu: 50m + # memory: 64Mi + # limits: + # cpu: 200m + # memory: 128Mi From 34b86ae6f7cba3e42322a30852e4bef89f52939d Mon Sep 17 00:00:00 2001 From: Ryota Date: Wed, 7 Jan 2026 18:44:26 +0000 Subject: [PATCH 29/58] Ensure to add backup volume --- .../controller/shard/containers.go | 27 +++++++++++++++++++ .../controller/shard/containers_test.go | 24 +++++++++++++++++ .../controller/shard/integration_test.go | 24 +++++++++++++++++ .../controller/shard/pool_statefulset.go | 1 + .../controller/shard/pool_statefulset_test.go | 4 +++ 5 files changed, 80 insertions(+) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 2e97df40..28402e20 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -50,6 +50,13 @@ const ( // SocketDirMountPath is the mount path for unix sockets (postgres and pgctld communicate here) SocketDirMountPath = "/var/run/sockets" + + // BackupVolumeName is the name of the backup volume for pgbackrest + BackupVolumeName = "backup-data" + + // BackupMountPath is where the backup volume is mounted + // pgbackrest stores backups here via --repo1-path + BackupMountPath = "/backups" ) // sidecarRestartPolicy is the restart policy for native sidecar containers @@ -109,6 +116,10 @@ func buildPostgresContainer( Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, } } @@ -176,6 +187,10 @@ func buildMultiPoolerSidecar( Name: DataVolumeName, // Shares PVC with postgres for pgbackrest configs and sockets MountPath: PoolerDirMountPath, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, } } @@ -243,3 +258,15 @@ func buildPgctldVolume() corev1.Volume { }, } } + +// buildBackupVolume creates the backup volume for pgbackrest. +// Uses emptyDir for development/testing. For production, this should be +// replaced with persistent storage (PVC or cloud storage). +func buildBackupVolume() corev1.Volume { + return corev1.Volume{ + Name: BackupVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 4b914224..1a065cf1 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -64,6 +64,10 @@ func TestBuildPostgresContainer(t *testing.T) { Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, @@ -117,6 +121,10 @@ func TestBuildPostgresContainer(t *testing.T) { Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, @@ -188,6 +196,10 @@ func TestBuildPostgresContainer(t *testing.T) { Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, @@ -272,6 +284,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: DataVolumeName, MountPath: PoolerDirMountPath, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, @@ -339,6 +355,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: DataVolumeName, MountPath: PoolerDirMountPath, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, @@ -424,6 +444,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: DataVolumeName, MountPath: PoolerDirMountPath, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, }, }, }, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 5bf724ff..1304e8dd 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -273,6 +273,7 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, @@ -305,6 +306,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, @@ -315,6 +317,12 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, }, }, @@ -541,6 +549,12 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, InitContainers: []corev1.Container{ { @@ -587,6 +601,7 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, @@ -619,6 +634,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, @@ -694,6 +710,12 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, InitContainers: []corev1.Container{ { @@ -740,6 +762,7 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, @@ -772,6 +795,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + {Name: "backup-data", MountPath: "/backups"}, }, }, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 4673ef28..ad0185a9 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -81,6 +81,7 @@ func BuildPoolStatefulSet( }, Volumes: []corev1.Volume{ buildPgctldVolume(), + buildBackupVolume(), // Single PVC shared by both postgres and multipooler because both need // access to pgbackrest configs, sockets, and postgres data directory }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index e1d2479f..636b635e 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -135,6 +135,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, Volumes: []corev1.Volume{ buildPgctldVolume(), + buildBackupVolume(), }, }, }, @@ -271,6 +272,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, Volumes: []corev1.Volume{ buildPgctldVolume(), + buildBackupVolume(), }, }, }, @@ -402,6 +404,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, Volumes: []corev1.Volume{ buildPgctldVolume(), + buildBackupVolume(), }, }, }, @@ -550,6 +553,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, Volumes: []corev1.Volume{ buildPgctldVolume(), + buildBackupVolume(), }, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ From 084d50855c74e2cf87076c0fe4a8e4758ca3afae Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 02:29:28 +0000 Subject: [PATCH 30/58] Use pgctld image directly rather than copying --- .../controller/shard/containers.go | 112 +++++++++++++++++- .../controller/shard/containers_test.go | 52 +++++--- .../controller/shard/integration_test.go | 99 +++++++--------- .../controller/shard/pool_statefulset.go | 12 +- .../controller/shard/pool_statefulset_test.go | 112 ++++++++++++++---- .../e2e-config/kind-shard.yaml | 2 +- 6 files changed, 280 insertions(+), 109 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 28402e20..96db29f5 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -13,19 +13,25 @@ const ( DefaultMultigresImage = "ghcr.io/multigres/multigres:main" // DefaultPgctldImage is the image containing the pgctld binary + // Used by buildPgctldInitContainer() for the binary-copy approach DefaultPgctldImage = "ghcr.io/multigres/pgctld:main" // DefaultPostgresImage is the default PostgreSQL database container image + // Used by buildPostgresContainer() for the original stock postgres:17 approach + // NOTE: Currently unused - buildPgctldContainer() uses ghcr.io/multigres/pgctld:main instead DefaultPostgresImage = "postgres:17" // PgctldVolumeName is the name of the shared volume for pgctld binary + // Used only by alternative approach (binary-copy via init container) PgctldVolumeName = "pgctld-bin" // PgctldBinDir is the directory where pgctld binary is mounted // Subdirectory avoids shadowing postgres binaries in /usr/local/bin + // Used only by alternative approach (binary-copy via init container) PgctldBinDir = "/usr/local/bin/multigres" // PgctldMountPath is the full path to pgctld binary + // Used only by alternative approach (binary-copy via init container) PgctldMountPath = PgctldBinDir + "/pgctld" // DataVolumeName is the name of the data volume for PostgreSQL @@ -49,7 +55,8 @@ const ( SocketDirVolumeName = "socket-dir" // SocketDirMountPath is the mount path for unix sockets (postgres and pgctld communicate here) - SocketDirMountPath = "/var/run/sockets" + // We use /var/run/postgresql because that is the default socket directory for the official postgres image. + SocketDirMountPath = "/var/run/postgresql" // BackupVolumeName is the name of the backup volume for pgbackrest BackupVolumeName = "backup-data" @@ -59,12 +66,27 @@ const ( BackupMountPath = "/backups" ) +// buildSocketDirVolume creates the shared emptyDir volume for unix sockets. +func buildSocketDirVolume() corev1.Volume { + return corev1.Volume{ + Name: SocketDirVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } +} + // sidecarRestartPolicy is the restart policy for native sidecar containers var sidecarRestartPolicy = corev1.ContainerRestartPolicyAlways // buildPostgresContainer creates the postgres container spec for a pool. -// Uses pgctld server instead of running postgres directly because pgctld provides -// lifecycle management (init, start, stop, status) via gRPC for multiorch/multipooler. +// ORIGINAL APPROACH: Uses stock postgres:17 image with pgctld binary copied via init container. +// Currently unused - replaced by buildPgctldContainer. Kept for reference. +// +// This approach requires: +// - buildPgctldInitContainer() in InitContainers +// - buildPgctldVolume() in Volumes +// - Does NOT include pgbackrest (would need manual installation) func buildPostgresContainer( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -116,10 +138,79 @@ func buildPostgresContainer( Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + }, + } +} + +// buildPgctldContainer creates the postgres container spec using the pgctld image. +// CURRENT APPROACH: Uses DefaultPgctldImage (ghcr.io/multigres/pgctld:main) which includes: +// - PostgreSQL 17 +// - pgctld binary at /usr/local/bin/pgctld +// - pgbackrest for backup/restore operations +// +// This approach does NOT require: +// - buildPgctldInitContainer() (pgctld already in image) +// - buildPgctldVolume() (no binary copying needed) +// +// Key differences from buildPostgresContainer: +// - Uses DefaultPgctldImage instead of DefaultPostgresImage (stock postgres:17) +// - Includes backup volume mount (for pgbackrest) +// - Includes socket dir volume mount (for shared sockets) +// - Does NOT include pgctld-bin volume mount +func buildPgctldContainer( + shard *multigresv1alpha1.Shard, + pool multigresv1alpha1.PoolSpec, +) corev1.Container { + image := DefaultPgctldImage + if shard.Spec.Images.Postgres != "" { + image = shard.Spec.Images.Postgres + } + + return corev1.Container{ + Name: "postgres", + Image: image, + Command: []string{"/usr/local/bin/pgctld"}, + Args: []string{ + "server", + "--pooler-dir=" + PoolerDirMountPath, + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + }, + Resources: pool.Postgres.Resources, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_HOST_AUTH_METHOD", + Value: "trust", + }, + { + Name: "PGDATA", + Value: PgDataPath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: DataVolumeName, + MountPath: DataMountPath, + }, { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, } } @@ -147,7 +238,7 @@ func buildMultiPoolerSidecar( "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", // Unix socket uses trust auth (no password) + "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", // Unix socket uses trust auth (no password) "--service-map", "grpc-pooler", // Only enable grpc-pooler service (disables auto-restore service) "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, @@ -191,12 +282,17 @@ func buildMultiPoolerSidecar( Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, } } // buildPgctldInitContainer creates the pgctld init container spec. -// Copies pgctld binary to shared volume because postgres image doesn't include it. +// ALTERNATIVE APPROACH: Copies pgctld binary to shared volume for stock postgres:17 image. +// Currently unused - kept for reference. Active approach uses pgctld image with built-in pgctld. func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { return corev1.Container{ Name: "pgctld-init", @@ -223,7 +319,6 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co // TODO: Add remaining command line arguments: // --log-level, --log-output, --hostname - // --cluster-metadata-refresh-interval, --pooler-health-check-interval, --recovery-cycle-interval // TODO: Verify correct format for --watch-targets flag. // Currently using static "postgres" based on demo, but may need to be: @@ -238,6 +333,9 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, "--cell", cellName, "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", } return corev1.Container{ @@ -250,6 +348,8 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } // buildPgctldVolume creates the shared emptyDir volume for pgctld binary. +// ALTERNATIVE APPROACH: Used only when copying pgctld via init container. +// Currently unused - kept for reference. Active approach uses pgctld image. func buildPgctldVolume() corev1.Volume { return corev1.Volume{ Name: PgctldVolumeName, diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 1a065cf1..4d2a28d0 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -26,7 +26,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: DefaultPostgresImage, - Command: []string{PgctldMountPath}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -60,14 +60,14 @@ func TestBuildPostgresContainer(t *testing.T) { Name: DataVolumeName, MountPath: DataMountPath, }, - { - Name: PgctldVolumeName, - MountPath: PgctldBinDir, - }, { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -83,7 +83,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: "postgres:16", - Command: []string{PgctldMountPath}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -117,14 +117,14 @@ func TestBuildPostgresContainer(t *testing.T) { Name: DataVolumeName, MountPath: DataMountPath, }, - { - Name: PgctldVolumeName, - MountPath: PgctldBinDir, - }, { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -149,7 +149,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: DefaultPostgresImage, - Command: []string{PgctldMountPath}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -192,14 +192,14 @@ func TestBuildPostgresContainer(t *testing.T) { Name: DataVolumeName, MountPath: DataMountPath, }, - { - Name: PgctldVolumeName, - MountPath: PgctldBinDir, - }, { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -249,7 +249,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", @@ -288,6 +288,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -320,7 +324,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", @@ -359,6 +363,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -400,7 +408,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", + "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", @@ -448,6 +456,10 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { Name: BackupVolumeName, MountPath: BackupMountPath, }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, }, }, }, @@ -529,6 +541,9 @@ func TestBuildMultiOrchContainer(t *testing.T) { "--topo-global-root", "/multigres/global", "--cell", "zone1", "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: buildMultiOrchContainerPorts(), Resources: corev1.ResourceRequirements{}, @@ -561,3 +576,4 @@ func TestBuildPgctldVolume(t *testing.T) { t.Errorf("buildPgctldVolume() mismatch (-want +got):\n%s", diff) } } + diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 1304e8dd..ec6285d4 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -233,17 +233,18 @@ func TestShardReconciliation(t *testing.T) { }, Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ - { - Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", - Command: []string{"/bin/sh", "-c"}, - Args: []string{ - "cp /usr/local/bin/pgctld /shared/pgctld", - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, - }, - }, + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-init", + // Image: "ghcr.io/multigres/pgctld:main", + // Command: []string{"/bin/sh", "-c"}, + // Args: []string{ + // "cp /usr/local/bin/pgctld /shared/pgctld", + // }, + // VolumeMounts: []corev1.VolumeMount{ + // {Name: "pgctld-bin", MountPath: "/shared"}, + // }, + // }, { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -281,7 +282,7 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", - Command: []string{"/usr/local/bin/multigres/pgctld"}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=/var/lib/pooler", @@ -305,18 +306,20 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, }, }, }, Volumes: []corev1.Volume{ - { - Name: "pgctld-bin", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, { Name: "backup-data", VolumeSource: corev1.VolumeSource{ @@ -543,12 +546,13 @@ func TestShardReconciliation(t *testing.T) { FSGroup: ptr.To(int64(999)), }, Volumes: []corev1.Volume{ - { - Name: "pgctld-bin", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, { Name: "backup-data", VolumeSource: corev1.VolumeSource{ @@ -557,17 +561,6 @@ func TestShardReconciliation(t *testing.T) { }, }, InitContainers: []corev1.Container{ - { - Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", - Command: []string{"/bin/sh", "-c"}, - Args: []string{ - "cp /usr/local/bin/pgctld /shared/pgctld", - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, - }, - }, { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -609,7 +602,7 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", - Command: []string{"/usr/local/bin/multigres/pgctld"}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=/var/lib/pooler", @@ -633,7 +626,8 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, }, }, @@ -704,12 +698,13 @@ func TestShardReconciliation(t *testing.T) { FSGroup: ptr.To(int64(999)), }, Volumes: []corev1.Volume{ - { - Name: "pgctld-bin", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, + // ALTERNATIVE: Uncomment for binary-copy approach + // { + // Name: "pgctld-bin", + // VolumeSource: corev1.VolumeSource{ + // EmptyDir: &corev1.EmptyDirVolumeSource{}, + // }, + // }, { Name: "backup-data", VolumeSource: corev1.VolumeSource{ @@ -718,17 +713,6 @@ func TestShardReconciliation(t *testing.T) { }, }, InitContainers: []corev1.Container{ - { - Name: "pgctld-init", - Image: "ghcr.io/multigres/pgctld:main", - Command: []string{"/bin/sh", "-c"}, - Args: []string{ - "cp /usr/local/bin/pgctld /shared/pgctld", - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "pgctld-bin", MountPath: "/shared"}, - }, - }, { Name: "multipooler", Image: "ghcr.io/multigres/multigres:main", @@ -770,7 +754,7 @@ func TestShardReconciliation(t *testing.T) { { Name: "postgres", Image: "postgres:17", - Command: []string{"/usr/local/bin/multigres/pgctld"}, + Command: []string{"/usr/local/bin/pgctld"}, Args: []string{ "server", "--pooler-dir=/var/lib/pooler", @@ -794,7 +778,8 @@ func TestShardReconciliation(t *testing.T) { }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, - {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, + // ALTERNATIVE: Uncomment for binary-copy approach + // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, }, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index ad0185a9..518cd5b1 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -73,15 +73,21 @@ func BuildPoolStatefulSet( FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(shard), + // ALTERNATIVE APPROACH: Uncomment below for stock postgres:17 + binary copy + // buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, Containers: []corev1.Container{ - buildPostgresContainer(shard, poolSpec), + // CURRENT: Uses ghcr.io/multigres/pgctld:main with built-in pgctld + pgbackrest + buildPgctldContainer(shard, poolSpec), + // ALTERNATIVE: Use stock postgres:17 + binary-copy approach + // buildPostgresContainer(shard, poolSpec), }, Volumes: []corev1.Volume{ - buildPgctldVolume(), + // ALTERNATIVE APPROACH: Uncomment below for binary-copy approach + // buildPgctldVolume(), buildBackupVolume(), + buildSocketDirVolume(), // Single PVC shared by both postgres and multipooler because both need // access to pgbackrest configs, sockets, and postgres data directory }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 636b635e..17370431 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -109,9 +109,6 @@ func TestBuildPoolStatefulSet(t *testing.T) { FSGroup: ptr.To(int64(999)), }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "test-shard"}, @@ -120,7 +117,12 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, "primary", "zone1", ), @@ -130,12 +132,17 @@ func TestBuildPoolStatefulSet(t *testing.T) { &multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{}, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, ), }, Volumes: []corev1.Volume{ - buildPgctldVolume(), buildBackupVolume(), + buildSocketDirVolume(), }, }, }, @@ -244,9 +251,6 @@ func TestBuildPoolStatefulSet(t *testing.T) { FSGroup: ptr.To(int64(999)), }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-001"}, @@ -256,7 +260,13 @@ func TestBuildPoolStatefulSet(t *testing.T) { }, }, multigresv1alpha1.PoolSpec{ - Cells: []multigresv1alpha1.CellName{"zone-west"}, + Type: "readOnly", + Cells: []multigresv1alpha1.CellName{"zone-west"}, + ReplicasPerCell: ptr.To(int32(3)), + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + Size: "20Gi", + }, }, "replica", "zone-west", @@ -267,12 +277,20 @@ func TestBuildPoolStatefulSet(t *testing.T) { &multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{}, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "readOnly", + Cells: []multigresv1alpha1.CellName{"zone-west"}, + ReplicasPerCell: ptr.To(int32(3)), + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + Size: "20Gi", + }, + }, ), }, Volumes: []corev1.Volume{ - buildPgctldVolume(), buildBackupVolume(), + buildSocketDirVolume(), }, }, }, @@ -378,9 +396,6 @@ func TestBuildPoolStatefulSet(t *testing.T) { FSGroup: ptr.To(int64(999)), }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-002"}, @@ -389,7 +404,11 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Size: "5Gi", + }, + }, "readOnly", "zone1", ), @@ -399,12 +418,16 @@ func TestBuildPoolStatefulSet(t *testing.T) { &multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{}, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Size: "5Gi", + }, + }, ), }, Volumes: []corev1.Volume{ - buildPgctldVolume(), buildBackupVolume(), + buildSocketDirVolume(), }, }, }, @@ -527,9 +550,6 @@ func TestBuildPoolStatefulSet(t *testing.T) { FSGroup: ptr.To(int64(999)), }, InitContainers: []corev1.Container{ - buildPgctldInitContainer(&multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, - }), buildMultiPoolerSidecar( &multigresv1alpha1.Shard{ ObjectMeta: metav1.ObjectMeta{Name: "shard-affinity"}, @@ -538,7 +558,29 @@ func TestBuildPoolStatefulSet(t *testing.T) { TableGroupName: "default", }, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "disk-type", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"ssd"}, + }, + }, + }, + }, + }, + }, + }, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, "primary", "zone1", ), @@ -548,12 +590,34 @@ func TestBuildPoolStatefulSet(t *testing.T) { &multigresv1alpha1.Shard{ Spec: multigresv1alpha1.ShardSpec{}, }, - multigresv1alpha1.PoolSpec{}, + multigresv1alpha1.PoolSpec{ + Type: "replica", + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "disk-type", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"ssd"}, + }, + }, + }, + }, + }, + }, + }, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, ), }, Volumes: []corev1.Volume{ - buildPgctldVolume(), buildBackupVolume(), + buildSocketDirVolume(), }, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 150a15b9..56a6c325 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -30,7 +30,7 @@ spec: images: multiorch: ghcr.io/multigres/multigres:main multipooler: ghcr.io/multigres/multigres:main - postgres: postgres:17 + postgres: ghcr.io/multigres/pgctld:main # Reference to the global topology server globalTopoServer: From 31e5d609dd5f2ad7b6fa853237e05af189c7433e Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 04:03:45 +0000 Subject: [PATCH 31/58] Correct socket, add pvc handling --- .../controller/shard/containers.go | 7 ++- .../controller/shard/pool_statefulset.go | 53 ++++++++++++++++++- .../controller/shard/pool_statefulset_test.go | 8 +-- .../controller/shard/shard_controller.go | 48 +++++++++++++++++ 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 96db29f5..6416a7d2 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -238,7 +238,7 @@ func buildMultiPoolerSidecar( "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", // Unix socket uses trust auth (no password) + "--socket-file", PoolerDirMountPath + "/pg_sockets/.s.PGSQL.5432", // Unix socket uses trust auth (no password) "--service-map", "grpc-pooler", // Only enable grpc-pooler service (disables auto-restore service) "--topo-global-server-addresses", shard.Spec.GlobalTopoServer.Address, "--topo-global-root", shard.Spec.GlobalTopoServer.RootPath, @@ -366,7 +366,10 @@ func buildBackupVolume() corev1.Volume { return corev1.Volume{ Name: BackupVolumeName, VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + HostPath: &corev1.HostPathVolumeSource{ + Path: "/data/backups", + Type: &pathType, + }, }, } } diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 518cd5b1..2c68daa3 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -5,6 +5,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -86,7 +87,7 @@ func BuildPoolStatefulSet( Volumes: []corev1.Volume{ // ALTERNATIVE APPROACH: Uncomment below for binary-copy approach // buildPgctldVolume(), - buildBackupVolume(), + buildBackupVolume(name, poolSpec.BackupStorageType), buildSocketDirVolume(), // Single PVC shared by both postgres and multipooler because both need // access to pgbackrest configs, sockets, and postgres data directory @@ -124,3 +125,53 @@ func buildPoolVolumeClaimTemplates( storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize), } } + +// BuildBackupPVC creates a standalone PVC for backup storage shared across all pods in a pool. +// This PVC is created independently of the StatefulSet and referenced by all pods. +func BuildBackupPVC( + shard *multigresv1alpha1.Shard, + poolName string, + cellName string, + poolSpec multigresv1alpha1.PoolSpec, + scheme *runtime.Scheme, +) (*corev1.PersistentVolumeClaim, error) { + name := buildPoolNameWithCell(shard.Name, poolName, cellName) + pvcName := "backup-data-" + name + labels := buildPoolLabelsWithCell(shard, poolName, cellName, poolSpec) + + var storageClass *string + storageSize := "10Gi" // Default backup storage size + + if poolSpec.Storage.Class != "" { + storageClass = &poolSpec.Storage.Class + } + // TODO: Add backup-specific storage size configuration to PoolSpec + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: shard.Namespace, + Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, // For single-node clusters like kind. Use ReadWriteMany for multi-node production. + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(storageSize), + }, + }, + }, + } + + if storageClass != nil { + pvc.Spec.StorageClassName = storageClass + } + + if err := ctrl.SetControllerReference(shard, pvc, scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + + return pvc, nil +} diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 17370431..800ff56b 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -141,7 +141,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume(), + buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), buildSocketDirVolume(), }, }, @@ -289,7 +289,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume(), + buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), buildSocketDirVolume(), }, }, @@ -426,7 +426,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume(), + buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), buildSocketDirVolume(), }, }, @@ -616,7 +616,7 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume(), + buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), buildSocketDirVolume(), }, Affinity: &corev1.Affinity{ diff --git a/pkg/resource-handler/controller/shard/shard_controller.go b/pkg/resource-handler/controller/shard/shard_controller.go index f8498838..ba8f41ad 100644 --- a/pkg/resource-handler/controller/shard/shard_controller.go +++ b/pkg/resource-handler/controller/shard/shard_controller.go @@ -221,6 +221,13 @@ func (r *ShardReconciler) reconcilePool( for _, cell := range poolSpec.Cells { cellName := string(cell) + // Reconcile backup PVC before StatefulSet (PVC must exist first) - only if using PVC storage + if poolSpec.BackupStorageType == "pvc" { + if err := r.reconcilePoolBackupPVC(ctx, shard, poolName, cellName, poolSpec); err != nil { + return fmt.Errorf("failed to reconcile backup PVC for cell %s: %w", cellName, err) + } + } + // Reconcile pool StatefulSet for this cell if err := r.reconcilePoolStatefulSet(ctx, shard, poolName, cellName, poolSpec); err != nil { return fmt.Errorf("failed to reconcile pool StatefulSet for cell %s: %w", cellName, err) @@ -279,6 +286,47 @@ func (r *ShardReconciler) reconcilePoolStatefulSet( return nil } +// reconcilePoolBackupPVC creates or updates the shared backup PVC for a pool in a specific cell. +func (r *ShardReconciler) reconcilePoolBackupPVC( + ctx context.Context, + shard *multigresv1alpha1.Shard, + poolName string, + cellName string, + poolSpec multigresv1alpha1.PoolSpec, +) error { + desired, err := BuildBackupPVC(shard, poolName, cellName, poolSpec, r.Scheme) + if err != nil { + return fmt.Errorf("failed to build backup PVC: %w", err) + } + + existing := &corev1.PersistentVolumeClaim{} + err = r.Get( + ctx, + client.ObjectKey{Namespace: shard.Namespace, Name: desired.Name}, + existing, + ) + if err != nil { + if errors.IsNotFound(err) { + // Create new PVC + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create backup PVC: %w", err) + } + return nil + } + return fmt.Errorf("failed to get backup PVC: %w", err) + } + + // PVCs are immutable after creation, only update labels/annotations if needed + if desired.Labels != nil { + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update backup PVC labels: %w", err) + } + } + + return nil +} + // reconcilePoolHeadlessService creates or updates the headless Service for a pool in a specific cell. func (r *ShardReconciler) reconcilePoolHeadlessService( ctx context.Context, From 6fd0b8e0b6276041ad8a58ac6364d945631c0050 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 04:15:00 +0000 Subject: [PATCH 32/58] Update kustomization version --- config/manager/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 9c62fb20..3b4a6bd3 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/numtide/multigres-operator - newTag: 9d6dd13-dirty + newTag: 084d508-dirty From 2d04650995857f90b3c579bd0e0f05df3e885bc1 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 04:15:18 +0000 Subject: [PATCH 33/58] Add pvc handling --- .../controller/shard/containers.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 6416a7d2..2bd4026c 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -360,9 +360,21 @@ func buildPgctldVolume() corev1.Volume { } // buildBackupVolume creates the backup volume for pgbackrest. -// Uses emptyDir for development/testing. For production, this should be -// replaced with persistent storage (PVC or cloud storage). -func buildBackupVolume() corev1.Volume { +// Uses either hostPath (for single-node testing) or PVC (for production) based on backupStorageType. +func buildBackupVolume(poolName string, backupStorageType string) corev1.Volume { + if backupStorageType == "pvc" { + return corev1.Volume{ + Name: BackupVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-" + poolName, + }, + }, + } + } + + // Default to hostPath for single-node testing (kind, minikube, etc.) + pathType := corev1.HostPathDirectoryOrCreate return corev1.Volume{ Name: BackupVolumeName, VolumeSource: corev1.VolumeSource{ From 219dc9bef2c6ab5e075a4ce112ef09cdd22f8fda Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 12:22:40 +0000 Subject: [PATCH 34/58] Remove unused env variable --- pkg/resource-handler/controller/shard/containers.go | 9 --------- .../controller/shard/containers_test.go | 12 ------------ .../controller/shard/integration_test.go | 3 --- 3 files changed, 24 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 2bd4026c..8a4bdbdb 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -114,11 +114,6 @@ func buildPostgresContainer( }, Resources: pool.Postgres.Resources, Env: []corev1.EnvVar{ - // NOTE: This is for MVP demo setup. - { - Name: "POSTGRES_HOST_AUTH_METHOD", - Value: "trust", - }, { Name: "PGDATA", Value: PgDataPath, @@ -184,10 +179,6 @@ func buildPgctldContainer( }, Resources: pool.Postgres.Resources, Env: []corev1.EnvVar{ - { - Name: "POSTGRES_HOST_AUTH_METHOD", - Value: "trust", - }, { Name: "PGDATA", Value: PgDataPath, diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 4d2a28d0..fc56d020 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -41,10 +41,6 @@ func TestBuildPostgresContainer(t *testing.T) { }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ - { - Name: "POSTGRES_HOST_AUTH_METHOD", - Value: "trust", - }, { Name: "PGDATA", Value: PgDataPath, @@ -98,10 +94,6 @@ func TestBuildPostgresContainer(t *testing.T) { }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ - { - Name: "POSTGRES_HOST_AUTH_METHOD", - Value: "trust", - }, { Name: "PGDATA", Value: PgDataPath, @@ -173,10 +165,6 @@ func TestBuildPostgresContainer(t *testing.T) { }, }, Env: []corev1.EnvVar{ - { - Name: "POSTGRES_HOST_AUTH_METHOD", - Value: "trust", - }, { Name: "PGDATA", Value: PgDataPath, diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index ec6285d4..b8ba1ac0 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -296,7 +296,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-socket-file=/var/lib/pooler/pgctld.sock", }, Env: []corev1.EnvVar{ - {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, }, SecurityContext: &corev1.SecurityContext{ @@ -616,7 +615,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-socket-file=/var/lib/pooler/pgctld.sock", }, Env: []corev1.EnvVar{ - {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, }, SecurityContext: &corev1.SecurityContext{ @@ -768,7 +766,6 @@ func TestShardReconciliation(t *testing.T) { "--grpc-socket-file=/var/lib/pooler/pgctld.sock", }, Env: []corev1.EnvVar{ - {Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"}, {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, }, SecurityContext: &corev1.SecurityContext{ From 50b595f4398bce1b4ec0ee8c31f8d9204e363a99 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 12:43:38 +0000 Subject: [PATCH 35/58] Add pb_hba.conf handling --- .../controller/shard/configmap.go | 55 +++++++++++++++++++ .../shard/templates/pg_hba_template.conf | 36 ++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 pkg/resource-handler/controller/shard/configmap.go create mode 100644 pkg/resource-handler/controller/shard/templates/pg_hba_template.conf diff --git a/pkg/resource-handler/controller/shard/configmap.go b/pkg/resource-handler/controller/shard/configmap.go new file mode 100644 index 00000000..12c51673 --- /dev/null +++ b/pkg/resource-handler/controller/shard/configmap.go @@ -0,0 +1,55 @@ +package shard + +import ( + _ "embed" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + + multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" +) + +// DefaultPgHbaTemplate is the default pg_hba.conf template for pooler instances. +// Uses trust authentication for testing/development. Production deployments should +// override this with proper authentication (scram-sha-256, SSL certificates, etc.). +// +//go:embed templates/pg_hba_template.conf +var DefaultPgHbaTemplate string + +// BuildPgHbaConfigMap creates a ConfigMap containing the pg_hba.conf template. +// This ConfigMap is shared across all pools in a shard and mounted into postgres containers. +func BuildPgHbaConfigMap( + shard *multigresv1alpha1.Shard, + scheme *runtime.Scheme, +) (*corev1.ConfigMap, error) { + // TODO: Add Shard.Spec.PgHbaTemplate field to allow custom templates + template := DefaultPgHbaTemplate + + labels := map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": shard.Name, + "app.kubernetes.io/component": "pg-hba-config", + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: PgHbaConfigMapName, + Namespace: shard.Namespace, + Labels: labels, + }, + Data: map[string]string{ + "pg_hba_template.conf": template, + }, + } + + if err := ctrl.SetControllerReference(shard, cm, scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + + return cm, nil +} diff --git a/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf b/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf new file mode 100644 index 00000000..2701ddbb --- /dev/null +++ b/pkg/resource-handler/controller/shard/templates/pg_hba_template.conf @@ -0,0 +1,36 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Generated by Multigres - Helping you Scale PostgreSQL +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all {{.User}} trust +local all all peer + +# Replication connections - trust from all IPs for testing +# PRODUCTION: Replace 'trust' with 'scram-sha-256' and configure passwords +local replication all trust +host replication all 0.0.0.0/0 trust +host replication all ::0/0 trust + +# IPv4 local connections: +host all all 0.0.0.0/0 trust +host all all ::0/0 trust + +# IPv4 external connections +host all all 0.0.0.0/0 trust +host all all ::0/0 trust + +# IPv6 external connections +host all all ::0/0 trust From 3c2eca12bc60dea77269c50a0c50125732326c74 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 12:44:47 +0000 Subject: [PATCH 36/58] Add pb_hba handling to container setup --- .../controller/shard/containers.go | 32 +++++++++++++++++++ .../controller/shard/containers_test.go | 18 +++++++++++ .../controller/shard/pool_statefulset.go | 1 + 3 files changed, 51 insertions(+) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 8a4bdbdb..1ec71597 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -64,6 +64,18 @@ const ( // BackupMountPath is where the backup volume is mounted // pgbackrest stores backups here via --repo1-path BackupMountPath = "/backups" + + // PgHbaConfigMapName is the name of the ConfigMap containing pg_hba template + PgHbaConfigMapName = "pg-hba-template" + + // PgHbaVolumeName is the name of the volume for pg_hba template + PgHbaVolumeName = "pg-hba-template" + + // PgHbaMountPath is where the pg_hba template is mounted + PgHbaMountPath = "/etc/pgctld" + + // PgHbaTemplatePath is the full path to the pg_hba template file + PgHbaTemplatePath = PgHbaMountPath + "/pg_hba_template.conf" ) // buildSocketDirVolume creates the shared emptyDir volume for unix sockets. @@ -76,6 +88,20 @@ func buildSocketDirVolume() corev1.Volume { } } +// buildPgHbaVolume creates the volume for pg_hba template from ConfigMap. +func buildPgHbaVolume() corev1.Volume { + return corev1.Volume{ + Name: PgHbaVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: PgHbaConfigMapName, + }, + }, + }, + } +} + // sidecarRestartPolicy is the restart policy for native sidecar containers var sidecarRestartPolicy = corev1.ContainerRestartPolicyAlways @@ -176,6 +202,7 @@ func buildPgctldContainer( "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: pool.Postgres.Resources, Env: []corev1.EnvVar{ @@ -202,6 +229,11 @@ func buildPgctldContainer( Name: SocketDirVolumeName, MountPath: SocketDirMountPath, }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, }, } } diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index fc56d020..813bd190 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -38,6 +38,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -64,6 +65,11 @@ func TestBuildPostgresContainer(t *testing.T) { Name: SocketDirVolumeName, MountPath: SocketDirMountPath, }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, }, }, }, @@ -91,6 +97,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -117,6 +124,11 @@ func TestBuildPostgresContainer(t *testing.T) { Name: SocketDirVolumeName, MountPath: SocketDirMountPath, }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, }, }, }, @@ -153,6 +165,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -188,6 +201,11 @@ func TestBuildPostgresContainer(t *testing.T) { Name: SocketDirVolumeName, MountPath: SocketDirMountPath, }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, }, }, }, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 2c68daa3..c9d05afb 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -89,6 +89,7 @@ func BuildPoolStatefulSet( // buildPgctldVolume(), buildBackupVolume(name, poolSpec.BackupStorageType), buildSocketDirVolume(), + buildPgHbaVolume(), // Single PVC shared by both postgres and multipooler because both need // access to pgbackrest configs, sockets, and postgres data directory }, From 2a196eef093ed718ff50e006c4d8a668b1e3be98 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 19:42:33 +0000 Subject: [PATCH 37/58] Add pg_hba setup and backup volume --- .../controller/shard/pool_statefulset_test.go | 12 +++-- .../controller/shard/shard_controller.go | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 800ff56b..60bdb192 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -141,8 +141,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), + buildBackupVolume("test-shard-pool-primary-zone1"), buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -289,8 +290,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), + buildBackupVolume("test-shard-pool-primary-zone1"), buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -426,8 +428,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), + buildBackupVolume("test-shard-pool-primary-zone1"), buildSocketDirVolume(), + buildPgHbaVolume(), }, }, }, @@ -616,8 +619,9 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1", "hostPath"), + buildBackupVolume("test-shard-pool-primary-zone1"), buildSocketDirVolume(), + buildPgHbaVolume(), }, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ diff --git a/pkg/resource-handler/controller/shard/shard_controller.go b/pkg/resource-handler/controller/shard/shard_controller.go index ba8f41ad..88ccd9b1 100644 --- a/pkg/resource-handler/controller/shard/shard_controller.go +++ b/pkg/resource-handler/controller/shard/shard_controller.go @@ -60,6 +60,12 @@ func (r *ShardReconciler) Reconcile( } } + // Reconcile pg_hba ConfigMap first (required by all pools before StatefulSets start) + if err := r.reconcilePgHbaConfigMap(ctx, shard); err != nil { + logger.Error(err, "Failed to reconcile pg_hba ConfigMap") + return ctrl.Result{}, err + } + // Reconcile MultiOrch - one Deployment and Service per cell multiOrchCells, err := getMultiOrchCells(shard) if err != nil { @@ -161,6 +167,44 @@ func (r *ShardReconciler) reconcileMultiOrchDeployment( return nil } +// reconcilePgHbaConfigMap creates or updates the pg_hba ConfigMap for a shard. +// This ConfigMap is shared across all pools and contains the authentication template. +func (r *ShardReconciler) reconcilePgHbaConfigMap( + ctx context.Context, + shard *multigresv1alpha1.Shard, +) error { + desired, err := BuildPgHbaConfigMap(shard, r.Scheme) + if err != nil { + return fmt.Errorf("failed to build pg_hba ConfigMap: %w", err) + } + + existing := &corev1.ConfigMap{} + err = r.Get( + ctx, + client.ObjectKey{Namespace: shard.Namespace, Name: desired.Name}, + existing, + ) + if err != nil { + if errors.IsNotFound(err) { + // Create new ConfigMap + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create pg_hba ConfigMap: %w", err) + } + return nil + } + return fmt.Errorf("failed to get pg_hba ConfigMap: %w", err) + } + + // Update existing ConfigMap + existing.Data = desired.Data + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update pg_hba ConfigMap: %w", err) + } + + return nil +} + // reconcileMultiOrchService creates or updates the MultiOrch Service for a specific cell. func (r *ShardReconciler) reconcileMultiOrchService( ctx context.Context, From f7c38a8811891a005e4fac7ff44d316389b55747 Mon Sep 17 00:00:00 2001 From: Ryota Date: Thu, 8 Jan 2026 20:06:07 +0000 Subject: [PATCH 38/58] Correct with PVC binding and update comments --- .../controller/shard/containers.go | 23 +++-------- .../controller/shard/pool_statefulset.go | 38 +++++++++++++------ .../controller/shard/shard_controller.go | 8 ++-- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 1ec71597..1e3871ef 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -383,27 +383,14 @@ func buildPgctldVolume() corev1.Volume { } // buildBackupVolume creates the backup volume for pgbackrest. -// Uses either hostPath (for single-node testing) or PVC (for production) based on backupStorageType. -func buildBackupVolume(poolName string, backupStorageType string) corev1.Volume { - if backupStorageType == "pvc" { - return corev1.Volume{ - Name: BackupVolumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "backup-data-" + poolName, - }, - }, - } - } - - // Default to hostPath for single-node testing (kind, minikube, etc.) - pathType := corev1.HostPathDirectoryOrCreate +// References a PVC that is created separately and shared across all pods in a pool. +// For single-node clusters (kind), ReadWriteOnce works since all pods are on the same node. +func buildBackupVolume(poolName string) corev1.Volume { return corev1.Volume{ Name: BackupVolumeName, VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/data/backups", - Type: &pathType, + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-" + poolName, }, }, } diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index c9d05afb..8bbd2399 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -74,24 +74,22 @@ func BuildPoolStatefulSet( FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image }, InitContainers: []corev1.Container{ - // ALTERNATIVE APPROACH: Uncomment below for stock postgres:17 + binary copy - // buildPgctldInitContainer(shard), + // ALTERNATIVE: Add init container to copy pgctld and pgbackrest binaries + // to emptyDir, enabling use of stock postgres:17 image + // buildBinaryCopyInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, Containers: []corev1.Container{ - // CURRENT: Uses ghcr.io/multigres/pgctld:main with built-in pgctld + pgbackrest buildPgctldContainer(shard, poolSpec), - // ALTERNATIVE: Use stock postgres:17 + binary-copy approach + // ALTERNATIVE: Use stock postgres:17 with copied binaries // buildPostgresContainer(shard, poolSpec), }, Volumes: []corev1.Volume{ - // ALTERNATIVE APPROACH: Uncomment below for binary-copy approach - // buildPgctldVolume(), - buildBackupVolume(name, poolSpec.BackupStorageType), + // ALTERNATIVE: Add emptyDir volume for binary copy + // buildBinariesVolume(), + buildBackupVolume(name), buildSocketDirVolume(), buildPgHbaVolume(), - // Single PVC shared by both postgres and multipooler because both need - // access to pgbackrest configs, sockets, and postgres data directory }, Affinity: poolSpec.Affinity, }, @@ -129,6 +127,8 @@ func buildPoolVolumeClaimTemplates( // BuildBackupPVC creates a standalone PVC for backup storage shared across all pods in a pool. // This PVC is created independently of the StatefulSet and referenced by all pods. +// For single-node clusters (kind, minikube), uses ReadWriteOnce (all pods on same node). +// For multi-node production, configure BackupStorage.Class to a storage class supporting ReadWriteMany. func BuildBackupPVC( shard *multigresv1alpha1.Shard, poolName string, @@ -140,13 +140,27 @@ func BuildBackupPVC( pvcName := "backup-data-" + name labels := buildPoolLabelsWithCell(shard, poolName, cellName, poolSpec) + // Use BackupStorage if specified, otherwise inherit from Storage var storageClass *string storageSize := "10Gi" // Default backup storage size - if poolSpec.Storage.Class != "" { + if poolSpec.BackupStorage.Class != "" { + storageClass = &poolSpec.BackupStorage.Class + } else if poolSpec.Storage.Class != "" { storageClass = &poolSpec.Storage.Class } - // TODO: Add backup-specific storage size configuration to PoolSpec + + if poolSpec.BackupStorage.Size != "" { + storageSize = poolSpec.BackupStorage.Size + } + + // Default to ReadWriteOnce for single-node clusters. + // TODO: When StorageSpec.AccessMode is added, use: + // accessMode := corev1.ReadWriteOnce + // if poolSpec.BackupStorage.AccessMode != "" { + // accessMode = poolSpec.BackupStorage.AccessMode + // } + accessMode := corev1.ReadWriteOnce pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -156,7 +170,7 @@ func BuildBackupPVC( }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, // For single-node clusters like kind. Use ReadWriteMany for multi-node production. + accessMode, }, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ diff --git a/pkg/resource-handler/controller/shard/shard_controller.go b/pkg/resource-handler/controller/shard/shard_controller.go index 88ccd9b1..7b8bda75 100644 --- a/pkg/resource-handler/controller/shard/shard_controller.go +++ b/pkg/resource-handler/controller/shard/shard_controller.go @@ -265,11 +265,9 @@ func (r *ShardReconciler) reconcilePool( for _, cell := range poolSpec.Cells { cellName := string(cell) - // Reconcile backup PVC before StatefulSet (PVC must exist first) - only if using PVC storage - if poolSpec.BackupStorageType == "pvc" { - if err := r.reconcilePoolBackupPVC(ctx, shard, poolName, cellName, poolSpec); err != nil { - return fmt.Errorf("failed to reconcile backup PVC for cell %s: %w", cellName, err) - } + // Reconcile backup PVC before StatefulSet (PVC must exist first) + if err := r.reconcilePoolBackupPVC(ctx, shard, poolName, cellName, poolSpec); err != nil { + return fmt.Errorf("failed to reconcile backup PVC for cell %s: %w", cellName, err) } // Reconcile pool StatefulSet for this cell From 76d27e3a5994845da120df67335aa20be59255ab Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 11:18:29 +0000 Subject: [PATCH 39/58] Add test and correct comment reference --- .../controller/shard/configmap_test.go | 168 ++++++++++++++++++ .../controller/shard/pool_statefulset.go | 4 +- 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 pkg/resource-handler/controller/shard/configmap_test.go diff --git a/pkg/resource-handler/controller/shard/configmap_test.go b/pkg/resource-handler/controller/shard/configmap_test.go new file mode 100644 index 00000000..cef2a472 --- /dev/null +++ b/pkg/resource-handler/controller/shard/configmap_test.go @@ -0,0 +1,168 @@ +package shard + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + multigresv1alpha1 "github.com/numtide/multigres-operator/api/v1alpha1" +) + +func TestBuildPgHbaConfigMap(t *testing.T) { + defaultScheme := runtime.NewScheme() + _ = multigresv1alpha1.AddToScheme(defaultScheme) + + tests := map[string]struct { + shard *multigresv1alpha1.Shard + scheme *runtime.Scheme + wantErr bool + }{ + "creates configmap with default template": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + UID: "test-uid", + }, + }, + wantErr: false, + }, + "creates configmap with correct embedded template": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "production-shard", + Namespace: "prod", + UID: "prod-uid", + }, + }, + wantErr: false, + }, + "returns error when scheme is invalid (missing Shard kind)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "error-shard", + Namespace: "default", + UID: "error-uid", + }, + }, + scheme: runtime.NewScheme(), // Empty scheme + wantErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + scheme := tc.scheme + if scheme == nil { + scheme = defaultScheme + } + + cm, err := BuildPgHbaConfigMap(tc.shard, scheme) + if (err != nil) != tc.wantErr { + t.Fatalf("BuildPgHbaConfigMap() error = %v, wantErr %v", err, tc.wantErr) + } + + if tc.wantErr { + return + } + + if cm.Name != PgHbaConfigMapName { + t.Errorf("ConfigMap name = %v, want %v", cm.Name, PgHbaConfigMapName) + } + if cm.Namespace != tc.shard.Namespace { + t.Errorf("ConfigMap namespace = %v, want %v", cm.Namespace, tc.shard.Namespace) + } + + // Verify owner reference + if len(cm.OwnerReferences) != 1 { + t.Fatalf("Expected 1 owner reference, got %d", len(cm.OwnerReferences)) + } + ownerRef := cm.OwnerReferences[0] + if ownerRef.Name != tc.shard.Name || ownerRef.Kind != "Shard" { + t.Errorf("Owner reference = %+v, want Shard/%s", ownerRef, tc.shard.Name) + } + if !ptr.Deref(ownerRef.Controller, false) { + t.Error("Expected owner reference to be controller") + } + + // Verify labels + expectedLabels := map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": tc.shard.Name, + "app.kubernetes.io/component": "pg-hba-config", + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + } + if diff := cmp.Diff(expectedLabels, cm.Labels); diff != "" { + t.Errorf("Labels mismatch (-want +got):\n%s", diff) + } + + // Verify template content exists + template, ok := cm.Data["pg_hba_template.conf"] + if !ok { + t.Fatal("ConfigMap missing pg_hba_template.conf key") + } + + // Verify the template matches what's embedded (source of truth) + if template != DefaultPgHbaTemplate { + t.Error("Template content doesn't match DefaultPgHbaTemplate") + } + }) + } +} + +func TestDefaultPgHbaTemplateEmbedded(t *testing.T) { + // Verify the embedded template is not empty + if DefaultPgHbaTemplate == "" { + t.Error("DefaultPgHbaTemplate is empty - go:embed may have failed") + } + + // Verify critical configuration lines exist + // We check for the presence of rules, ignoring multiple spaces + checks := []struct { + desc string + mustContain []string + }{ + { + desc: "header", + mustContain: []string{"# PostgreSQL Client Authentication"}, + }, + { + desc: "local trust rule", + mustContain: []string{"local", "all", "{{.User}}", "trust"}, + }, + { + desc: "replication trust rule", + mustContain: []string{"host", "replication", "all", "0.0.0.0/0", "trust"}, + }, + { + desc: "production warning", + mustContain: []string{"PRODUCTION:", "scram-sha-256"}, + }, + } + + for _, check := range checks { + found := false + lines := strings.Split(DefaultPgHbaTemplate, "\n") + for _, line := range lines { + allMatch := true + for _, part := range check.mustContain { + if !strings.Contains(line, part) { + allMatch = false + break + } + } + if allMatch { + found = true + break + } + } + if !found { + t.Errorf("DefaultPgHbaTemplate missing %s (expected line containing all of: %v)", check.desc, check.mustContain) + } + } +} diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 8bbd2399..f8e18df1 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -76,7 +76,7 @@ func BuildPoolStatefulSet( InitContainers: []corev1.Container{ // ALTERNATIVE: Add init container to copy pgctld and pgbackrest binaries // to emptyDir, enabling use of stock postgres:17 image - // buildBinaryCopyInitContainer(shard), + // buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, Containers: []corev1.Container{ @@ -86,7 +86,7 @@ func BuildPoolStatefulSet( }, Volumes: []corev1.Volume{ // ALTERNATIVE: Add emptyDir volume for binary copy - // buildBinariesVolume(), + // buildPgctldVolume(), buildBackupVolume(name), buildSocketDirVolume(), buildPgHbaVolume(), From 7f48a65fdec69415305a8a914680d981773aebc9 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 11:38:40 +0000 Subject: [PATCH 40/58] Update kustomization tag --- config/manager/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 3b4a6bd3..32c8f89c 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/numtide/multigres-operator - newTag: 084d508-dirty + newTag: 3c2eca1-dirty From 721950a0516cb5e19916c1f66e37e63e1a313898 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 14:09:13 +0000 Subject: [PATCH 41/58] Update api reference --- pkg/resource-handler/go.mod | 2 +- pkg/resource-handler/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resource-handler/go.mod b/pkg/resource-handler/go.mod index 47629df9..b8a7e6af 100644 --- a/pkg/resource-handler/go.mod +++ b/pkg/resource-handler/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/google/go-cmp v0.7.0 - github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b + github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805 github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373 k8s.io/api v0.34.3 k8s.io/apimachinery v0.34.3 diff --git a/pkg/resource-handler/go.sum b/pkg/resource-handler/go.sum index 964f3275..f5773f55 100644 --- a/pkg/resource-handler/go.sum +++ b/pkg/resource-handler/go.sum @@ -90,8 +90,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b h1:twErev/AdyVQvlYa5vbo5wqq+NlexZJmIm0pwEQd5qI= -github.com/numtide/multigres-operator/api v0.0.0-20260103121224-057738b43b3b/go.mod h1:A1bBmTxHr+362dGZ5G6u2S4xsP6enbgdUS/UJUOmKbc= +github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805 h1:XUmBsLf2Lv1AIRkPLN8ipn/QBnOlJ7SGu1RidhovhwY= +github.com/numtide/multigres-operator/api v0.0.0-20260109130614-8d663cc4b805/go.mod h1:A1bBmTxHr+362dGZ5G6u2S4xsP6enbgdUS/UJUOmKbc= github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373 h1:B9uGjUsG0rMi+dGt2blEDpr8wbwnE/W1xcpuxZwvOYk= github.com/numtide/multigres-operator/pkg/testutil v0.0.0-20251213002906-55493b734373/go.mod h1:+NQa7dSvQqxhBOE9XcE9RWXLvOvNaw0keCc29Y7pjyQ= github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= From 5cc1f8f615b385c526d7dfcb67c4ba8e4a48069c Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:46:31 +0000 Subject: [PATCH 42/58] Add explicit access modes --- pkg/resource-handler/e2e-config/kind-cell.yaml | 2 ++ pkg/resource-handler/e2e-config/kind-shard.yaml | 2 ++ pkg/resource-handler/e2e-config/kind-toposerver.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/pkg/resource-handler/e2e-config/kind-cell.yaml b/pkg/resource-handler/e2e-config/kind-cell.yaml index 377be80a..3861de08 100644 --- a/pkg/resource-handler/e2e-config/kind-cell.yaml +++ b/pkg/resource-handler/e2e-config/kind-cell.yaml @@ -44,6 +44,8 @@ spec: replicas: 1 storage: size: 1Gi + accessModes: + - ReadWriteOnce resources: requests: cpu: 50m diff --git a/pkg/resource-handler/e2e-config/kind-shard.yaml b/pkg/resource-handler/e2e-config/kind-shard.yaml index 56a6c325..efcb5499 100644 --- a/pkg/resource-handler/e2e-config/kind-shard.yaml +++ b/pkg/resource-handler/e2e-config/kind-shard.yaml @@ -63,6 +63,8 @@ spec: # Storage configuration - kind uses local storage provisioner storage: size: 1Gi + accessModes: + - ReadWriteOnce # PostgreSQL container configuration - lower resources for kind postgres: diff --git a/pkg/resource-handler/e2e-config/kind-toposerver.yaml b/pkg/resource-handler/e2e-config/kind-toposerver.yaml index 838cdc57..dda76279 100644 --- a/pkg/resource-handler/e2e-config/kind-toposerver.yaml +++ b/pkg/resource-handler/e2e-config/kind-toposerver.yaml @@ -21,6 +21,8 @@ spec: # Storage configuration for etcd data - kind uses local storage provisioner storage: size: 1Gi + accessModes: + - ReadWriteOnce # Resource requirements for etcd containers - lower for kind resources: From 6bfa5f02a69bfd9c3d6aa37c2ce5babf1892efd6 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:51:02 +0000 Subject: [PATCH 43/58] Add access mode handling for PVCs --- .../controller/shard/pool_statefulset.go | 19 +++++++++---------- .../controller/storage/pvc.go | 12 +++++++++--- .../controller/toposerver/statefulset.go | 4 +++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index f8e18df1..102fa877 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -121,7 +121,7 @@ func buildPoolVolumeClaimTemplates( } return []corev1.PersistentVolumeClaim{ - storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize), + storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize, pool.Storage.AccessModes), } } @@ -155,12 +155,13 @@ func BuildBackupPVC( } // Default to ReadWriteOnce for single-node clusters. - // TODO: When StorageSpec.AccessMode is added, use: - // accessMode := corev1.ReadWriteOnce - // if poolSpec.BackupStorage.AccessMode != "" { - // accessMode = poolSpec.BackupStorage.AccessMode - // } - accessMode := corev1.ReadWriteOnce + accessModes := []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + } + + if len(poolSpec.BackupStorage.AccessModes) > 0 { + accessModes = poolSpec.BackupStorage.AccessModes + } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -169,9 +170,7 @@ func BuildBackupPVC( Labels: labels, }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - accessMode, - }, + AccessModes: accessModes, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse(storageSize), diff --git a/pkg/resource-handler/controller/storage/pvc.go b/pkg/resource-handler/controller/storage/pvc.go index 84be0773..cc1b113f 100644 --- a/pkg/resource-handler/controller/storage/pvc.go +++ b/pkg/resource-handler/controller/storage/pvc.go @@ -15,19 +15,25 @@ import ( // - name: Name for the volume claim (e.g., "data") // - storageClassName: Optional storage class name (nil uses cluster default) // - storageSize: Size of the volume (e.g., "10Gi") +// - accessModes: List of access modes (e.g., [ReadWriteOnce]). Defaults to ReadWriteOnce if empty. func BuildPVCTemplate( name string, storageClassName *string, storageSize string, + accessModes []corev1.PersistentVolumeAccessMode, ) corev1.PersistentVolumeClaim { + if len(accessModes) == 0 { + accessModes = []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + } + } + pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, + AccessModes: accessModes, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse(storageSize), diff --git a/pkg/resource-handler/controller/toposerver/statefulset.go b/pkg/resource-handler/controller/toposerver/statefulset.go index 3a43717f..a6263c72 100644 --- a/pkg/resource-handler/controller/toposerver/statefulset.go +++ b/pkg/resource-handler/controller/toposerver/statefulset.go @@ -120,6 +120,7 @@ func buildVolumeClaimTemplates( ) []corev1.PersistentVolumeClaim { var storageClass *string storageSize := DefaultStorageSize + var accessModes []corev1.PersistentVolumeAccessMode if toposerver.Spec.Etcd != nil { if toposerver.Spec.Etcd.Storage.Class != "" { @@ -128,9 +129,10 @@ func buildVolumeClaimTemplates( if toposerver.Spec.Etcd.Storage.Size != "" { storageSize = toposerver.Spec.Etcd.Storage.Size } + accessModes = toposerver.Spec.Etcd.Storage.AccessModes } return []corev1.PersistentVolumeClaim{ - storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize), + storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize, accessModes), } } From c8a032df51c09ebbb9cd7b3185858b72958b2f92 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:55:59 +0000 Subject: [PATCH 44/58] Correct pgctld mount path --- .../controller/shard/containers_test.go | 45 ++++--------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 813bd190..2946a603 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -26,7 +26,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: DefaultPostgresImage, - Command: []string{"/usr/local/bin/pgctld"}, + Command: []string{"/usr/local/bin/multigres/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -58,17 +58,8 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: BackupVolumeName, - MountPath: BackupMountPath, - }, - { - Name: SocketDirVolumeName, - MountPath: SocketDirMountPath, - }, - { - Name: PgHbaVolumeName, - MountPath: PgHbaMountPath, - ReadOnly: true, + Name: "pgctld-bin", + MountPath: "/usr/local/bin/multigres", }, }, }, @@ -85,7 +76,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: "postgres:16", - Command: []string{"/usr/local/bin/pgctld"}, + Command: []string{"/usr/local/bin/multigres/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -117,17 +108,8 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: BackupVolumeName, - MountPath: BackupMountPath, - }, - { - Name: SocketDirVolumeName, - MountPath: SocketDirMountPath, - }, - { - Name: PgHbaVolumeName, - MountPath: PgHbaMountPath, - ReadOnly: true, + Name: "pgctld-bin", + MountPath: "/usr/local/bin/multigres", }, }, }, @@ -153,7 +135,7 @@ func TestBuildPostgresContainer(t *testing.T) { want: corev1.Container{ Name: "postgres", Image: DefaultPostgresImage, - Command: []string{"/usr/local/bin/pgctld"}, + Command: []string{"/usr/local/bin/multigres/pgctld"}, Args: []string{ "server", "--pooler-dir=" + PoolerDirMountPath, @@ -194,17 +176,8 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: BackupVolumeName, - MountPath: BackupMountPath, - }, - { - Name: SocketDirVolumeName, - MountPath: SocketDirMountPath, - }, - { - Name: PgHbaVolumeName, - MountPath: PgHbaMountPath, - ReadOnly: true, + Name: "pgctld-bin", + MountPath: "/usr/local/bin/multigres", }, }, }, From 8074979fd68c8c711ed350164afc7d8229620747 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:56:10 +0000 Subject: [PATCH 45/58] Correct socket path --- pkg/resource-handler/controller/shard/containers_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 2946a603..6c4495a1 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -228,7 +228,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", @@ -303,7 +303,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", @@ -387,7 +387,7 @@ func TestBuildMultiPoolerSidecar(t *testing.T) { "--http-port", "15200", "--grpc-port", "15270", "--pooler-dir", PoolerDirMountPath, - "--socket-file", SocketDirMountPath + "/.s.PGSQL.5432", + "--socket-file", "/var/lib/pooler/pg_sockets/.s.PGSQL.5432", "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", From 56764f845df87bf408c7b92c6a7a916a4f5227ef Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:56:26 +0000 Subject: [PATCH 46/58] Correct zone --- .../controller/shard/integration_test.go | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index b8ba1ac0..a8755567 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -79,11 +79,11 @@ func TestShardReconciliation(t *testing.T) { Implementation: "etcd2", }, MultiOrch: multigresv1alpha1.MultiOrchSpec{ - Cells: []multigresv1alpha1.CellName{"us-west-1a", "us-west-1b"}, + Cells: []multigresv1alpha1.CellName{"zone-a", "zone-b"}, }, Pools: map[string]multigresv1alpha1.PoolSpec{ "primary": { - Cells: []multigresv1alpha1.CellName{"us-west-1a"}, + Cells: []multigresv1alpha1.CellName{"zone-a"}, Type: "readWrite", ReplicasPerCell: ptr.To(int32(2)), Storage: multigresv1alpha1.StorageSpec{ @@ -94,22 +94,22 @@ func TestShardReconciliation(t *testing.T) { }, }, wantResources: []client.Object{ - // MultiOrch Deployment for us-west-1a + // MultiOrch Deployment for zone-a &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.DeploymentSpec{ Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + MatchLabels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -122,7 +122,7 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--cell", "us-west-1a", + "--cell", "zone-a", "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ @@ -135,12 +135,12 @@ func TestShardReconciliation(t *testing.T) { }, }, }, - // MultiOrch Service for us-west-1a + // MultiOrch Service for zone-a &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Labels: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -149,25 +149,25 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "http", 15300), tcpServicePort(t, "grpc", 15370), }, - Selector: shardLabels(t, "test-shard-multiorch-us-west-1a", "multiorch", "us-west-1a"), + Selector: shardLabels(t, "test-shard-multiorch-zone-a", "multiorch", "zone-a"), }, }, - // MultiOrch Deployment for us-west-1b + // MultiOrch Deployment for zone-b &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1b", + Name: "test-shard-multiorch-zone-b", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.DeploymentSpec{ Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + MatchLabels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -180,7 +180,7 @@ func TestShardReconciliation(t *testing.T) { "--grpc-port", "15370", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--cell", "us-west-1b", + "--cell", "zone-b", "--watch-targets", "postgres", }, Ports: []corev1.ContainerPort{ @@ -193,12 +193,12 @@ func TestShardReconciliation(t *testing.T) { }, }, }, - // MultiOrch Service for us-west-1b + // MultiOrch Service for zone-b &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1b", + Name: "test-shard-multiorch-zone-b", Namespace: "default", - Labels: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Labels: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -207,21 +207,21 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "http", 15300), tcpServicePort(t, "grpc", 15370), }, - Selector: shardLabels(t, "test-shard-multiorch-us-west-1b", "multiorch", "us-west-1b"), + Selector: shardLabels(t, "test-shard-multiorch-zone-b", "multiorch", "zone-b"), }, }, &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-pool-primary-us-west-1a", + Name: "test-shard-pool-primary-zone-a", Namespace: "default", - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: appsv1.StatefulSetSpec{ - ServiceName: "test-shard-pool-primary-us-west-1a-headless", + ServiceName: "test-shard-pool-primary-zone-a-headless", Replicas: ptr.To(int32(2)), Selector: &metav1.LabelSelector{ - MatchLabels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + MatchLabels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), }, PodManagementPolicy: appsv1.ParallelPodManagement, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ @@ -229,7 +229,7 @@ func TestShardReconciliation(t *testing.T) { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), }, Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ @@ -257,7 +257,7 @@ func TestShardReconciliation(t *testing.T) { "--service-map", "grpc-pooler", "--topo-global-server-addresses", "global-topo:2379", "--topo-global-root", "/multigres/global", - "--cell", "us-west-1a", + "--cell", "zone-a", "--database", "testdb", "--table-group", "default", "--shard", "0", @@ -351,9 +351,9 @@ func TestShardReconciliation(t *testing.T) { }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-pool-primary-us-west-1a-headless", + Name: "test-shard-pool-primary-zone-a-headless", Namespace: "default", - Labels: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Labels: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), OwnerReferences: shardOwnerRefs(t, "test-shard"), }, Spec: corev1.ServiceSpec{ @@ -364,7 +364,7 @@ func TestShardReconciliation(t *testing.T) { tcpServicePort(t, "grpc", 15270), tcpServicePort(t, "postgres", 5432), }, - Selector: shardLabels(t, "test-shard-pool-primary-us-west-1a", "shard-pool", "us-west-1a"), + Selector: shardLabels(t, "test-shard-pool-primary-zone-a", "shard-pool", "zone-a"), PublishNotReadyAddresses: true, }, }, From dc21184ec53793a238d82282d5a382d79b089b4e Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:58:17 +0000 Subject: [PATCH 47/58] Correct socket dir input --- pkg/resource-handler/controller/shard/integration_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index a8755567..95a4c878 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -275,6 +275,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, @@ -308,6 +309,7 @@ func TestShardReconciliation(t *testing.T) { // ALTERNATIVE: Uncomment for binary-copy approach // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, @@ -594,6 +596,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, @@ -627,6 +630,7 @@ func TestShardReconciliation(t *testing.T) { // ALTERNATIVE: Uncomment for binary-copy approach // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, @@ -745,6 +749,7 @@ func TestShardReconciliation(t *testing.T) { VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, @@ -778,6 +783,7 @@ func TestShardReconciliation(t *testing.T) { // ALTERNATIVE: Uncomment for binary-copy approach // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, + {Name: "socket-dir", MountPath: "/var/run/postgresql"}, }, }, }, From 8fd7addbc74fe5814290a889d89bdfb25d7eaf25 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 19:59:37 +0000 Subject: [PATCH 48/58] Correct pg_hba template --- .../controller/shard/containers_test.go | 4 -- .../controller/shard/integration_test.go | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 6c4495a1..479a330c 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -38,7 +38,6 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", - "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -88,7 +87,6 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", - "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -147,7 +145,6 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", - "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -555,4 +552,3 @@ func TestBuildPgctldVolume(t *testing.T) { t.Errorf("buildPgctldVolume() mismatch (-want +got):\n%s", diff) } } - diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 95a4c878..d0ac9ac9 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -295,6 +295,7 @@ func TestShardReconciliation(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", }, Env: []corev1.EnvVar{ {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, @@ -310,6 +311,7 @@ func TestShardReconciliation(t *testing.T) { // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, @@ -327,6 +329,17 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, + }, + }, }, }, }, @@ -560,6 +573,17 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, + }, + }, }, InitContainers: []corev1.Container{ { @@ -616,6 +640,7 @@ func TestShardReconciliation(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", }, Env: []corev1.EnvVar{ {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, @@ -631,6 +656,7 @@ func TestShardReconciliation(t *testing.T) { // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, @@ -713,6 +739,17 @@ func TestShardReconciliation(t *testing.T) { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "pg-hba-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "pg-hba-template", + }, + DefaultMode: ptr.To(int32(420)), + }, + }, + }, }, InitContainers: []corev1.Container{ { @@ -769,6 +806,7 @@ func TestShardReconciliation(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", }, Env: []corev1.EnvVar{ {Name: "PGDATA", Value: "/var/lib/pooler/pg_data"}, @@ -784,6 +822,7 @@ func TestShardReconciliation(t *testing.T) { // {Name: "pgctld-bin", MountPath: "/usr/local/bin/multigres"}, {Name: "backup-data", MountPath: "/backups"}, {Name: "socket-dir", MountPath: "/var/run/postgresql"}, + {Name: "pg-hba-template", MountPath: "/etc/pgctld", ReadOnly: true}, }, }, }, From aa5e2fc90dd8062b293e6c0efd3ff8911afe7ef7 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 20:00:10 +0000 Subject: [PATCH 49/58] Correct zone in multiorch testing --- .../controller/shard/multiorch_test.go | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/resource-handler/controller/shard/multiorch_test.go b/pkg/resource-handler/controller/shard/multiorch_test.go index e430eb60..88c7ad31 100644 --- a/pkg/resource-handler/controller/shard/multiorch_test.go +++ b/pkg/resource-handler/controller/shard/multiorch_test.go @@ -41,23 +41,23 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Implementation: "etcd2", }, MultiOrch: multigresv1alpha1.MultiOrchSpec{ - Cells: []multigresv1alpha1.CellName{"us-west-1a"}, + Cells: []multigresv1alpha1.CellName{"zone-a"}, }, }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: scheme, want: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -77,11 +77,11 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -90,11 +90,11 @@ func TestBuildMultiOrchDeployment(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -109,7 +109,7 @@ func TestBuildMultiOrchDeployment(t *testing.T) { Implementation: "etcd2", }, }, - }, "us-west-1a"), + }, "zone-a"), }, }, }, @@ -315,7 +315,7 @@ func TestBuildMultiOrchDeployment(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: runtime.NewScheme(), // empty scheme wantErr: true, }, @@ -364,19 +364,19 @@ func TestBuildMultiOrchService(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: scheme, want: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-multiorch-us-west-1a", + Name: "test-shard-multiorch-zone-a", Namespace: "default", Labels: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -395,11 +395,11 @@ func TestBuildMultiOrchService(t *testing.T) { Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/name": "multigres", - "app.kubernetes.io/instance": "test-shard-multiorch-us-west-1a", + "app.kubernetes.io/instance": "test-shard-multiorch-zone-a", "app.kubernetes.io/component": MultiOrchComponentName, "app.kubernetes.io/part-of": "multigres", "app.kubernetes.io/managed-by": "multigres-operator", - "multigres.com/cell": "us-west-1a", + "multigres.com/cell": "zone-a", "multigres.com/database": "testdb", "multigres.com/tablegroup": "default", }, @@ -499,7 +499,7 @@ func TestBuildMultiOrchService(t *testing.T) { TableGroupName: "default", }, }, - cellName: "us-west-1a", + cellName: "zone-a", scheme: runtime.NewScheme(), // empty scheme wantErr: true, }, From 18c96b61aa314ec1576d06dd1e92e517d31f5632 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 20:00:24 +0000 Subject: [PATCH 50/58] Add access mode testing --- .../controller/storage/pvc_test.go | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/resource-handler/controller/storage/pvc_test.go b/pkg/resource-handler/controller/storage/pvc_test.go index 42c3b960..f6918cff 100644 --- a/pkg/resource-handler/controller/storage/pvc_test.go +++ b/pkg/resource-handler/controller/storage/pvc_test.go @@ -17,6 +17,7 @@ func TestBuildPVCTemplate(t *testing.T) { name string storageClassName *string storageSize string + accessModes []corev1.PersistentVolumeAccessMode want corev1.PersistentVolumeClaim }{ "basic case with storage class and size": { @@ -122,11 +123,35 @@ func TestBuildPVCTemplate(t *testing.T) { }, }, }, + "custom access modes": { + name: "backup-data", + storageClassName: &storageClassStandard, + storageSize: "50Gi", + accessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + want: corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + StorageClassName: &storageClassStandard, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("50Gi"), + }, + }, + }, + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { - got := BuildPVCTemplate(tc.name, tc.storageClassName, tc.storageSize) + got := BuildPVCTemplate(tc.name, tc.storageClassName, tc.storageSize, tc.accessModes) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("BuildPVCTemplate() mismatch (-want +got):\n%s", diff) } From 2f1a6da04878498a5447910e0e3213f5a5f3416f Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 20:00:56 +0000 Subject: [PATCH 51/58] Add backup volume --- .../controller/shard/integration_test.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index d0ac9ac9..40e3ee4e 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -272,6 +272,17 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, @@ -325,6 +336,14 @@ func TestShardReconciliation(t *testing.T) { // }, { Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-test-shard-pool-primary-zone-a", + }, + }, + }, + { + Name: "socket-dir", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -569,6 +588,14 @@ func TestShardReconciliation(t *testing.T) { // }, { Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-multi-cell-shard-pool-primary-zone1", + }, + }, + }, + { + Name: "socket-dir", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -617,6 +644,17 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, @@ -735,6 +773,14 @@ func TestShardReconciliation(t *testing.T) { // }, { Name: "backup-data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "backup-data-multi-cell-shard-pool-primary-zone2", + }, + }, + }, + { + Name: "socket-dir", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -783,6 +829,17 @@ func TestShardReconciliation(t *testing.T) { RunAsGroup: ptr.To(int64(999)), RunAsNonRoot: ptr.To(true), }, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ {Name: "pgdata", MountPath: "/var/lib/pooler"}, {Name: "backup-data", MountPath: "/backups"}, From 55306c1b746bf8ee8364f955391498d9103ba266 Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 20:01:41 +0000 Subject: [PATCH 52/58] Adjust test spec for postgres container --- .../controller/shard/pool_statefulset_test.go | 243 ++++++++++++++---- 1 file changed, 190 insertions(+), 53 deletions(-) diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 60bdb192..15593390 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -128,17 +128,56 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", }, - multigresv1alpha1.PoolSpec{ - Type: "replica", - Storage: multigresv1alpha1.StorageSpec{ - Size: "10Gi", + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ buildBackupVolume("test-shard-pool-primary-zone1"), @@ -274,23 +313,59 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", }, - multigresv1alpha1.PoolSpec{ - Type: "readOnly", - Cells: []multigresv1alpha1.CellName{"zone-west"}, - ReplicasPerCell: ptr.To(int32(3)), - Storage: multigresv1alpha1.StorageSpec{ - Class: "fast-ssd", - Size: "20Gi", + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1"), + buildBackupVolume("shard-001-pool-replica-zone-west"), buildSocketDirVolume(), buildPgHbaVolume(), }, @@ -416,19 +491,59 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", }, - multigresv1alpha1.PoolSpec{ - Storage: multigresv1alpha1.StorageSpec{ - Size: "5Gi", + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, }, - ), + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, + }, + }, + }, }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1"), + buildBackupVolume("shard-002-pool-readOnly-zone1"), buildSocketDirVolume(), buildPgHbaVolume(), }, @@ -589,37 +704,59 @@ func TestBuildPoolStatefulSet(t *testing.T) { ), }, Containers: []corev1.Container{ - buildPostgresContainer( - &multigresv1alpha1.Shard{ - Spec: multigresv1alpha1.ShardSpec{}, + { + Name: "postgres", + Image: DefaultPgctldImage, + Command: []string{ + "/usr/local/bin/pgctld", }, - multigresv1alpha1.PoolSpec{ - Type: "replica", - Affinity: &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: "disk-type", - Operator: corev1.NodeSelectorOpIn, - Values: []string{"ssd"}, - }, - }, - }, - }, - }, - }, + Args: []string{ + "server", + "--pooler-dir=/var/lib/pooler", + "--grpc-port=15470", + "--pg-port=5432", + "--pg-listen-addresses=*", + "--pg-database=postgres", + "--pg-user=postgres", + "--timeout=30", + "--log-level=info", + "--grpc-socket-file=/var/lib/pooler/pgctld.sock", + "--pg-hba-template=/etc/pgctld/pg_hba_template.conf", + }, + Env: []corev1.EnvVar{ + { + Name: "PGDATA", + Value: "/var/lib/pooler/pg_data", }, - Storage: multigresv1alpha1.StorageSpec{ - Size: "10Gi", + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "pgdata", + MountPath: "/var/lib/pooler", + }, + { + Name: "backup-data", + MountPath: "/backups", + }, + { + Name: "socket-dir", + MountPath: "/var/run/postgresql", + }, + { + Name: "pg-hba-template", + MountPath: "/etc/pgctld", + ReadOnly: true, }, }, - ), + }, }, Volumes: []corev1.Volume{ - buildBackupVolume("test-shard-pool-primary-zone1"), + buildBackupVolume("shard-affinity-pool-primary-zone1"), buildSocketDirVolume(), buildPgHbaVolume(), }, From eae66ea649beaa4ca668df7e98d7d09d6d97f2ac Mon Sep 17 00:00:00 2001 From: Ryota Date: Fri, 9 Jan 2026 20:04:08 +0000 Subject: [PATCH 53/58] Add all flags in integration test results --- .../controller/shard/integration_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/resource-handler/controller/shard/integration_test.go b/pkg/resource-handler/controller/shard/integration_test.go index 40e3ee4e..55efa795 100644 --- a/pkg/resource-handler/controller/shard/integration_test.go +++ b/pkg/resource-handler/controller/shard/integration_test.go @@ -124,6 +124,9 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-root", "/multigres/global", "--cell", "zone-a", "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -182,6 +185,9 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-root", "/multigres/global", "--cell", "zone-b", "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -470,6 +476,9 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-root", "/multigres/global", "--cell", "zone1", "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), @@ -528,6 +537,9 @@ func TestShardReconciliation(t *testing.T) { "--topo-global-root", "/multigres/global", "--cell", "zone2", "--watch-targets", "postgres", + "--cluster-metadata-refresh-interval", "500ms", + "--pooler-health-check-interval", "500ms", + "--recovery-cycle-interval", "500ms", }, Ports: []corev1.ContainerPort{ tcpPort(t, "http", 15300), From b8040f2f9e421ebd269d80eeafaad23da11b6d6a Mon Sep 17 00:00:00 2001 From: Ryota Date: Sat, 10 Jan 2026 01:26:47 +0000 Subject: [PATCH 54/58] Add backup PVC related test cases --- .../controller/shard/pool_statefulset_test.go | 190 +++++++++++ .../shard/shard_controller_internal_test.go | 16 + .../controller/shard/shard_controller_test.go | 315 +++++++++++++++++- 3 files changed, 510 insertions(+), 11 deletions(-) diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index 15593390..c734eb48 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -931,3 +931,193 @@ func TestBuildPoolVolumeClaimTemplates(t *testing.T) { }) } } + +func TestBuildBackupPVC(t *testing.T) { + scheme := runtime.NewScheme() + _ = multigresv1alpha1.AddToScheme(scheme) + + shard := &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + UID: "test-uid", + }, + } + + tests := map[string]struct { + poolSpec multigresv1alpha1.PoolSpec + want *corev1.PersistentVolumeClaim + }{ + "defaults": { + poolSpec: multigresv1alpha1.PoolSpec{}, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + "inherit storage class": { + poolSpec: multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + }, + }, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("fast-ssd"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + "override backup storage": { + poolSpec: multigresv1alpha1.PoolSpec{ + Storage: multigresv1alpha1.StorageSpec{ + Class: "fast-ssd", + }, + BackupStorage: multigresv1alpha1.StorageSpec{ + Class: "backup-nfs", + Size: "100Gi", + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + }, + want: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pool-primary-zone1", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "multigres", + "app.kubernetes.io/instance": "test-shard-pool-primary-zone1", + "app.kubernetes.io/component": PoolComponentName, + "app.kubernetes.io/part-of": "multigres", + "app.kubernetes.io/managed-by": "multigres-operator", + "multigres.com/cell": "zone1", + "multigres.com/database": "", + "multigres.com/tablegroup": "", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "multigres.com/v1alpha1", + Kind: "Shard", + Name: "test-shard", + UID: "test-uid", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("backup-nfs"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("100Gi"), + }, + }, + }, + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + testScheme := scheme + if name == "error on controller reference" { + testScheme = runtime.NewScheme() // Empty scheme triggers SetControllerReference error + } + + got, err := BuildBackupPVC( + shard, + "primary", + "zone1", + tc.poolSpec, + testScheme, + ) + + if name == "error on controller reference" { + if err == nil { + t.Error("BuildBackupPVC() expected error, got nil") + } + return + } + + if err != nil { + t.Errorf("BuildBackupPVC() error = %v", err) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("BuildBackupPVC() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestBuildBackupPVC_Error(t *testing.T) { + // Separate test for error case to avoid messing with table driven test logic too much + shard := &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + } + // Empty scheme causes SetControllerReference to fail + _, err := BuildBackupPVC(shard, "pool", "cell", multigresv1alpha1.PoolSpec{}, runtime.NewScheme()) + if err == nil { + t.Error("BuildBackupPVC() expected error with empty scheme, got nil") + } +} diff --git a/pkg/resource-handler/controller/shard/shard_controller_internal_test.go b/pkg/resource-handler/controller/shard/shard_controller_internal_test.go index 6bdc97f2..36cb57b7 100644 --- a/pkg/resource-handler/controller/shard/shard_controller_internal_test.go +++ b/pkg/resource-handler/controller/shard/shard_controller_internal_test.go @@ -188,6 +188,22 @@ func TestReconcile_InvalidScheme(t *testing.T) { return r.reconcilePoolHeadlessService(ctx, shard, "pool1", "", poolSpec) }, }, + "PoolBackupPVC": { + setupShard: func() *multigresv1alpha1.Shard { + return &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard", + Namespace: "default", + }, + } + }, + reconcileFunc: func(r *ShardReconciler, ctx context.Context, shard *multigresv1alpha1.Shard) error { + poolSpec := multigresv1alpha1.PoolSpec{ + Cells: []multigresv1alpha1.CellName{"cell1"}, + } + return r.reconcilePoolBackupPVC(ctx, shard, "pool1", "cell1", poolSpec) + }, + }, } for name, tc := range tests { diff --git a/pkg/resource-handler/controller/shard/shard_controller_test.go b/pkg/resource-handler/controller/shard/shard_controller_test.go index 46cca7df..50067d5d 100644 --- a/pkg/resource-handler/controller/shard/shard_controller_test.go +++ b/pkg/resource-handler/controller/shard/shard_controller_test.go @@ -28,11 +28,12 @@ func TestShardReconciler_Reconcile(t *testing.T) { _ = corev1.AddToScheme(scheme) tests := map[string]struct { - shard *multigresv1alpha1.Shard - existingObjects []client.Object - failureConfig *testutil.FailureConfig - wantErr bool - assertFunc func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) + shard *multigresv1alpha1.Shard + existingObjects []client.Object + failureConfig *testutil.FailureConfig + reconcilerScheme *runtime.Scheme + wantErr bool + assertFunc func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) }{ ////---------------------------------------- /// Success @@ -403,6 +404,18 @@ func TestShardReconciler_Reconcile(t *testing.T) { Namespace: "default", }, }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-hba-template", + Namespace: "default", + }, + }, + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-existing-shard-pool-primary-zone1", + Namespace: "default", + }, + }, }, assertFunc: func(t *testing.T, c client.Client, shard *multigresv1alpha1.Shard) { poolSts := &appsv1.StatefulSet{} @@ -1460,33 +1473,310 @@ func TestShardReconciler_Reconcile(t *testing.T) { existingObjects: []client.Object{ &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-multiorch", + Name: "test-shard-status-multiorch-zone1", Namespace: "default", }, }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-multiorch", + Name: "test-shard-status-multiorch-zone1", Namespace: "default", }, }, &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-pool-primary", + Name: "test-shard-status-pool-primary-zone1", Namespace: "default", }, }, &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-shard-status-pool-primary-headless", + Name: "test-shard-status-pool-primary-zone1-headless", Namespace: "default", }, }, }, failureConfig: &testutil.FailureConfig{ // Fail Pool StatefulSet Get after successful reconciliation calls - // Get calls: 1=Shard, 2=MultiOrchDeploy, 3=MultiOrchSvc, 4=PoolSts, 5=PoolSvc, 6=PoolSts(status) - OnGet: testutil.FailKeyAfterNCalls(5, testutil.ErrNetworkTimeout), + // Get calls: 1=Shard, 2=PgHbaCM, 3=MultiOrchDeploy, 4=MultiOrchSvc, 5=PoolBackupPVC, 6=PoolSts, 7=PoolSvc, 8=PoolSts(status) + OnGet: testutil.FailKeyAfterNCalls(7, testutil.ErrNetworkTimeout), + }, + wantErr: true, + }, + "error on build PgHba ConfigMap (empty scheme)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + reconcilerScheme: runtime.NewScheme(), // Empty scheme + wantErr: true, + }, + "error on Get PgHba ConfigMap (network error)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba-get-err", + Namespace: "default", + }, + }, + failureConfig: &testutil.FailureConfig{ + OnGet: func(key client.ObjectKey) error { + if key.Name == "pg-hba-template" { + return testutil.ErrNetworkTimeout + } + return nil + }, + }, + wantErr: true, + }, + "error on build MultiOrch Deployment (scheme missing Shard)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-mo-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + }, + }, + reconcilerScheme: func() *runtime.Scheme { + s := runtime.NewScheme() + // Register types but NOT Shard to fail SetControllerReference + _ = corev1.AddToScheme(s) + _ = appsv1.AddToScheme(s) + return s + }(), + wantErr: true, + }, + "error on build Pool BackupPVC (scheme missing Shard)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-pvc-err-shard", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + reconcilerScheme: func() *runtime.Scheme { + s := runtime.NewScheme() + _ = corev1.AddToScheme(s) + _ = appsv1.AddToScheme(s) + // Shard is NOT added + return s + }(), + wantErr: true, + }, + "error on PgHba ConfigMap create": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnCreate: func(obj client.Object) error { + if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Name == "pg-hba-template" { + return testutil.ErrPermissionError + } + return nil + }, + }, + wantErr: true, + }, + "error on PgHba ConfigMap update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pghba", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + }, + }, + existingObjects: []client.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg-hba-template", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Name == "pg-hba-template" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC create": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnCreate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrPermissionError + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{ + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pvc-pool-primary-zone1", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Pool BackupPVC labels update": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc-label-err", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + }, + }, + }, + existingObjects: []client.Object{ + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-data-test-shard-pvc-label-err-pool-primary-zone1", + Namespace: "default", + }, + }, + }, + failureConfig: &testutil.FailureConfig{ + OnUpdate: func(obj client.Object) error { + if pvc, ok := obj.(*corev1.PersistentVolumeClaim); ok && + pvc.Name == "backup-data-test-shard-pvc-label-err-pool-primary-zone1" { + return testutil.ErrInjected + } + return nil + }, + }, + wantErr: true, + }, + "error on Get Pool BackupPVC (network error)": { + shard: &multigresv1alpha1.Shard{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shard-pvc", + Namespace: "default", + }, + Spec: multigresv1alpha1.ShardSpec{ + DatabaseName: "testdb", + TableGroupName: "default", + MultiOrch: multigresv1alpha1.MultiOrchSpec{ + Cells: []multigresv1alpha1.CellName{"zone1"}, + }, + Pools: map[string]multigresv1alpha1.PoolSpec{ + "primary": { + Cells: []multigresv1alpha1.CellName{"zone1"}, + Storage: multigresv1alpha1.StorageSpec{ + Size: "10Gi", + }, + }, + }, + }, + }, + existingObjects: []client.Object{}, + failureConfig: &testutil.FailureConfig{ + OnGet: func(key client.ObjectKey) error { + if key.Name == "backup-data-test-shard-pvc-pool-primary-zone1" { + return testutil.ErrNetworkTimeout + } + return nil + }, }, wantErr: true, }, @@ -1513,6 +1803,9 @@ func TestShardReconciler_Reconcile(t *testing.T) { Client: fakeClient, Scheme: scheme, } + if tc.reconcilerScheme != nil { + reconciler.Scheme = tc.reconcilerScheme + } // Create the Shard resource if not in existing objects shardInExisting := false From e3e20b2fc0933a7172c513a9a1666f6d6cebda82 Mon Sep 17 00:00:00 2001 From: Ryota Date: Sat, 10 Jan 2026 01:31:06 +0000 Subject: [PATCH 55/58] Fix lint --- config/manager/kustomization.yaml | 8 ++++---- pkg/resource-handler/controller/shard/configmap_test.go | 6 +++++- pkg/resource-handler/controller/shard/pool_statefulset.go | 7 ++++++- .../controller/shard/pool_statefulset_test.go | 8 +++++++- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index bdac64df..03b86f2b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,8 +1,8 @@ resources: - - manager.yaml +- manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - - name: controller - newName: ghcr.io/numtide/multigres-operator - newTag: 3c2eca1-dirty +- name: controller + newName: ghcr.io/numtide/multigres-operator + newTag: a96d950 diff --git a/pkg/resource-handler/controller/shard/configmap_test.go b/pkg/resource-handler/controller/shard/configmap_test.go index cef2a472..e3f40577 100644 --- a/pkg/resource-handler/controller/shard/configmap_test.go +++ b/pkg/resource-handler/controller/shard/configmap_test.go @@ -162,7 +162,11 @@ func TestDefaultPgHbaTemplateEmbedded(t *testing.T) { } } if !found { - t.Errorf("DefaultPgHbaTemplate missing %s (expected line containing all of: %v)", check.desc, check.mustContain) + t.Errorf( + "DefaultPgHbaTemplate missing %s (expected line containing all of: %v)", + check.desc, + check.mustContain, + ) } } } diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 102fa877..86641bb9 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -121,7 +121,12 @@ func buildPoolVolumeClaimTemplates( } return []corev1.PersistentVolumeClaim{ - storage.BuildPVCTemplate(DataVolumeName, storageClass, storageSize, pool.Storage.AccessModes), + storage.BuildPVCTemplate( + DataVolumeName, + storageClass, + storageSize, + pool.Storage.AccessModes, + ), } } diff --git a/pkg/resource-handler/controller/shard/pool_statefulset_test.go b/pkg/resource-handler/controller/shard/pool_statefulset_test.go index c734eb48..61396012 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset_test.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset_test.go @@ -1116,7 +1116,13 @@ func TestBuildBackupPVC_Error(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "test"}, } // Empty scheme causes SetControllerReference to fail - _, err := BuildBackupPVC(shard, "pool", "cell", multigresv1alpha1.PoolSpec{}, runtime.NewScheme()) + _, err := BuildBackupPVC( + shard, + "pool", + "cell", + multigresv1alpha1.PoolSpec{}, + runtime.NewScheme(), + ) if err == nil { t.Error("BuildBackupPVC() expected error with empty scheme, got nil") } From f3d7bb0e8fbd88199cba3ea4fae612e3e685c9cd Mon Sep 17 00:00:00 2001 From: Ryota Date: Sat, 10 Jan 2026 01:44:51 +0000 Subject: [PATCH 56/58] Correct copy logic and comment --- .../controller/shard/containers.go | 39 +++++++++++-------- .../controller/shard/pool_statefulset.go | 6 +-- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers.go b/pkg/resource-handler/controller/shard/containers.go index 1e3871ef..0b12f878 100644 --- a/pkg/resource-handler/controller/shard/containers.go +++ b/pkg/resource-handler/controller/shard/containers.go @@ -106,13 +106,11 @@ func buildPgHbaVolume() corev1.Volume { var sidecarRestartPolicy = corev1.ContainerRestartPolicyAlways // buildPostgresContainer creates the postgres container spec for a pool. -// ORIGINAL APPROACH: Uses stock postgres:17 image with pgctld binary copied via init container. -// Currently unused - replaced by buildPgctldContainer. Kept for reference. +// Uses stock postgres:17 image with pgctld and pgbackrest binaries copied via init container. // // This approach requires: -// - buildPgctldInitContainer() in InitContainers +// - buildPgctldInitContainer() in InitContainers (copies pgctld and pgbackrest) // - buildPgctldVolume() in Volumes -// - Does NOT include pgbackrest (would need manual installation) func buildPostgresContainer( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -137,6 +135,7 @@ func buildPostgresContainer( "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: pool.Postgres.Resources, Env: []corev1.EnvVar{ @@ -159,12 +158,25 @@ func buildPostgresContainer( Name: PgctldVolumeName, MountPath: PgctldBinDir, }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, + }, }, } } // buildPgctldContainer creates the postgres container spec using the pgctld image. -// CURRENT APPROACH: Uses DefaultPgctldImage (ghcr.io/multigres/pgctld:main) which includes: +// Uses DefaultPgctldImage (ghcr.io/multigres/pgctld:main) which includes: // - PostgreSQL 17 // - pgctld binary at /usr/local/bin/pgctld // - pgbackrest for backup/restore operations @@ -172,12 +184,6 @@ func buildPostgresContainer( // This approach does NOT require: // - buildPgctldInitContainer() (pgctld already in image) // - buildPgctldVolume() (no binary copying needed) -// -// Key differences from buildPostgresContainer: -// - Uses DefaultPgctldImage instead of DefaultPostgresImage (stock postgres:17) -// - Includes backup volume mount (for pgbackrest) -// - Includes socket dir volume mount (for shared sockets) -// - Does NOT include pgctld-bin volume mount func buildPgctldContainer( shard *multigresv1alpha1.Shard, pool multigresv1alpha1.PoolSpec, @@ -314,15 +320,15 @@ func buildMultiPoolerSidecar( } // buildPgctldInitContainer creates the pgctld init container spec. -// ALTERNATIVE APPROACH: Copies pgctld binary to shared volume for stock postgres:17 image. -// Currently unused - kept for reference. Active approach uses pgctld image with built-in pgctld. +// Copies pgctld and pgbackrest binaries to shared volume for use with stock postgres:17 image. +// Used with buildPostgresContainer() and buildPgctldVolume(). func buildPgctldInitContainer(shard *multigresv1alpha1.Shard) corev1.Container { return corev1.Container{ Name: "pgctld-init", Image: DefaultPgctldImage, Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /usr/local/bin/pgctld /shared/pgctld", + "cp /usr/local/bin/pgctld /usr/bin/pgbackrest /shared/", }, VolumeMounts: []corev1.VolumeMount{ { @@ -370,9 +376,8 @@ func buildMultiOrchContainer(shard *multigresv1alpha1.Shard, cellName string) co } } -// buildPgctldVolume creates the shared emptyDir volume for pgctld binary. -// ALTERNATIVE APPROACH: Used only when copying pgctld via init container. -// Currently unused - kept for reference. Active approach uses pgctld image. +// buildPgctldVolume creates the shared emptyDir volume for pgctld and pgbackrest binaries. +// Used with buildPgctldInitContainer() and buildPostgresContainer(). func buildPgctldVolume() corev1.Volume { return corev1.Volume{ Name: PgctldVolumeName, diff --git a/pkg/resource-handler/controller/shard/pool_statefulset.go b/pkg/resource-handler/controller/shard/pool_statefulset.go index 86641bb9..d6afb104 100644 --- a/pkg/resource-handler/controller/shard/pool_statefulset.go +++ b/pkg/resource-handler/controller/shard/pool_statefulset.go @@ -74,18 +74,16 @@ func BuildPoolStatefulSet( FSGroup: ptr.To(int64(999)), // postgres group in postgres:17 image }, InitContainers: []corev1.Container{ - // ALTERNATIVE: Add init container to copy pgctld and pgbackrest binaries - // to emptyDir, enabling use of stock postgres:17 image + // To use stock postgres:17 image instead, uncomment buildPgctldInitContainer, + // replace buildPgctldContainer with buildPostgresContainer, and add buildPgctldVolume. // buildPgctldInitContainer(shard), buildMultiPoolerSidecar(shard, poolSpec, poolName, cellName), }, Containers: []corev1.Container{ buildPgctldContainer(shard, poolSpec), - // ALTERNATIVE: Use stock postgres:17 with copied binaries // buildPostgresContainer(shard, poolSpec), }, Volumes: []corev1.Volume{ - // ALTERNATIVE: Add emptyDir volume for binary copy // buildPgctldVolume(), buildBackupVolume(name), buildSocketDirVolume(), From 7c815474f794b22040331476080b43471a85b35e Mon Sep 17 00:00:00 2001 From: Ryota Date: Sat, 10 Jan 2026 01:47:42 +0000 Subject: [PATCH 57/58] Match test cases with updated impl --- .../controller/shard/containers_test.go | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/resource-handler/controller/shard/containers_test.go b/pkg/resource-handler/controller/shard/containers_test.go index 479a330c..5bf87389 100644 --- a/pkg/resource-handler/controller/shard/containers_test.go +++ b/pkg/resource-handler/controller/shard/containers_test.go @@ -38,6 +38,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -57,8 +58,21 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: "pgctld-bin", - MountPath: "/usr/local/bin/multigres", + Name: PgctldVolumeName, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -87,6 +101,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{}, Env: []corev1.EnvVar{ @@ -106,8 +121,21 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: "pgctld-bin", - MountPath: "/usr/local/bin/multigres", + Name: PgctldVolumeName, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -145,6 +173,7 @@ func TestBuildPostgresContainer(t *testing.T) { "--timeout=30", "--log-level=info", "--grpc-socket-file=" + PoolerDirMountPath + "/pgctld.sock", + "--pg-hba-template=" + PgHbaTemplatePath, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -173,8 +202,21 @@ func TestBuildPostgresContainer(t *testing.T) { MountPath: DataMountPath, }, { - Name: "pgctld-bin", - MountPath: "/usr/local/bin/multigres", + Name: PgctldVolumeName, + MountPath: PgctldBinDir, + }, + { + Name: BackupVolumeName, + MountPath: BackupMountPath, + }, + { + Name: SocketDirVolumeName, + MountPath: SocketDirMountPath, + }, + { + Name: PgHbaVolumeName, + MountPath: PgHbaMountPath, + ReadOnly: true, }, }, }, @@ -466,7 +508,7 @@ func TestBuildPgctldInitContainer(t *testing.T) { Image: DefaultPgctldImage, Command: []string{"/bin/sh", "-c"}, Args: []string{ - "cp /usr/local/bin/pgctld /shared/pgctld", + "cp /usr/local/bin/pgctld /usr/bin/pgbackrest /shared/", }, VolumeMounts: []corev1.VolumeMount{ { From e686f25b9084883ce0e2f1a215eb5a60227aae5a Mon Sep 17 00:00:00 2001 From: Ryota Date: Sat, 10 Jan 2026 01:49:51 +0000 Subject: [PATCH 58/58] Remove old files --- pkg/resource-handler/e2e-config/README.md | 54 --------------- .../e2e-config/register-topology.sh | 66 ------------------- 2 files changed, 120 deletions(-) delete mode 100644 pkg/resource-handler/e2e-config/README.md delete mode 100755 pkg/resource-handler/e2e-config/register-topology.sh diff --git a/pkg/resource-handler/e2e-config/README.md b/pkg/resource-handler/e2e-config/README.md deleted file mode 100644 index 36688f0b..00000000 --- a/pkg/resource-handler/e2e-config/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# E2E Configuration Files - -This directory contains sample CRDs for end-to-end testing and local development. - -## Files - -- **kind-cell.yaml**: Cell resource definition -- **kind-shard.yaml**: Shard resource definition -- **kind-toposerver.yaml**: TopoServer resource definition -- **register-topology.sh**: Script to register topology metadata - -## MVP Configuration - -The current configurations are set up for the MVP (Minimum Viable Product) with strict requirements: - -- Database must be `"postgres"` -- TableGroup must be `"default"` -- Shard name must be `"0-inf"` (unsharded range mode) - -Original e2e configurations with custom database/tablegroup names are commented out in the YAML files. - -## Topology Registration (Required Manual Step) - -**Important**: The `resource-handler` operator only manages Kubernetes resources (Deployments, Services, StatefulSets). It does NOT register cells and databases in the topology server (etcd). This will be implemented in the `data-handler` module. - -### Manual Registration for Testing - -After deploying the resources, you must manually register topology metadata: - -```bash -./register-topology.sh zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml -``` - -**All 5 arguments are required**: -1. Cells (comma-separated for multiple: `zone-a,zone-b`) -2. Database name -3. TopoServer service name -4. Namespace -5. Kubeconfig path - -This script uses the `multigres createclustermetadata` command to properly encode topology data as Protocol Buffers. - -### Without Registration - -If you skip this step, multipooler will crash with: -``` -Error: failed to get existing multipooler: Code: UNAVAILABLE -unable to get connection for cell "zone-a" -``` - -## Architecture - -- **resource-handler**: Manages Kubernetes resources (Deployments, Services, StatefulSets) ✅ -- **data-handler**: Will manage topology/data plane operations using multigres CLI/library ⏳ diff --git a/pkg/resource-handler/e2e-config/register-topology.sh b/pkg/resource-handler/e2e-config/register-topology.sh deleted file mode 100755 index c2d87b1a..00000000 --- a/pkg/resource-handler/e2e-config/register-topology.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# Helper script to register cells and database in the topology server -# Uses the multigres CLI to properly encode topology data as protobuf. -# -# Usage: ./register-topology.sh -# Example: ./register-topology.sh zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml - -set -euo pipefail - -if [ "$#" -ne 5 ]; then - echo "Error: Requires exactly 5 arguments" - echo "Usage: $0 " - echo "Example: $0 zone-a postgres kind-toposerver-sample default /tmp/kind-kubeconfig.yaml" - echo "" - echo "Note: For multiple cells, use comma-separated: zone-a,zone-b" - exit 1 -fi - -CELLS="$1" -DATABASE="$2" -TOPO_SERVICE="$3" -NAMESPACE="$4" -KUBE_CONFIG="$5" - -GLOBAL_ROOT="/multigres/global" -TOPO_ADDRESS="${TOPO_SERVICE}:2379" - -echo "Registering topology metadata..." -echo " Cells: ${CELLS}" -echo " Database: ${DATABASE}" -echo " Topo Service: ${TOPO_ADDRESS}" -echo " Namespace: ${NAMESPACE}" -echo " Kubeconfig: ${KUBE_CONFIG}" -echo "" - -# Find a pod with the multigres binary to run the command -# Try multiorch pod first, fallback to any pod with multigres image -EXEC_POD=$(kubectl --kubeconfig="$KUBE_CONFIG" get pods -n "$NAMESPACE" \ - -l app.kubernetes.io/component=multiorch \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) - -if [ -z "$EXEC_POD" ]; then - echo "Error: No multiorch pod found. Make sure Shard resources are deployed first." - exit 1 -fi - -echo "Using pod '${EXEC_POD}' to run multigres CLI..." -echo "" - -# Run createclustermetadata command -kubectl --kubeconfig="$KUBE_CONFIG" exec -n "$NAMESPACE" "$EXEC_POD" -- \ - /multigres/bin/multigres createclustermetadata \ - --global-topo-address="$TOPO_ADDRESS" \ - --global-topo-root="$GLOBAL_ROOT" \ - --cells="$CELLS" \ - --backup-location=/backup \ - --durability-policy=none - -echo "" -echo "✓ Topology registered successfully" -echo "" -echo "Verify cell registration:" -echo " kubectl --kubeconfig=$KUBE_CONFIG exec -n $NAMESPACE ${EXEC_POD} -- /multigres/bin/multigres getcellnames --global-topo-address=$TOPO_ADDRESS --global-topo-root=$GLOBAL_ROOT" -echo "" -echo "Verify database registration:" -echo " kubectl --kubeconfig=$KUBE_CONFIG exec -n $NAMESPACE ${EXEC_POD} -- /multigres/bin/multigres getdatabasenames --global-topo-address=$TOPO_ADDRESS --global-topo-root=$GLOBAL_ROOT"