enable updates to probes in gateways (#4901)

* demo code

* Update Makefile for local development; enhance deployment probe handling

* Create 4901.txt

* Update Consul image to use correct version

* Add support for Kubernetes health probes in managed Gateway classes

- Introduced configuration for liveness, readiness, and startup probes in GatewayClassConfig.
- Updated related templates and values to handle probe configurations.
- Enhanced command to load probes from a config map.

* Add tests for probe propagation and sanitization in GatewayClassConfig

* Improve error handling and sanitization in loadProbesConfig function

* Refactor Gateway Probes Configuration

Introduced a new mechanism to extract probe configurations directly from Gateway annotations.

* Remove unused transformGatewayClassConfig function from GatewayController

* Fix failing unit tests

* Fix comment formatting in ProbesConfig definition

* Refactor serializeGatewayClassConfig to ensure annotation reflects the latest GatewayClassConfig

Author: sanikachavan5

.changelog/4901.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+api-gateway: Add support for configuring Kubernetes probes (liveness, readiness, startup) per-Gateway via annotations. Use `consul.hashicorp.com/liveness-probe`, `consul.hashicorp.com/readiness-probe`, and `consul.hashicorp.com/startup-probe` annotations with JSON probe configuration to customize health checks for individual API Gateways. [[GH-4901](https://github.com/hashicorp/consul-k8s/pull/4901)]
+```

Makefile

@@ -55,7 +55,7 @@ control-plane-dev-docker: ## Build consul-k8s-control-plane dev Docker image.
        --build-arg 'GIT_COMMIT=$(GIT_COMMIT)' \
        --build-arg 'GIT_DIRTY=$(GIT_DIRTY)' \
        --build-arg 'GIT_DESCRIBE=$(GIT_DESCRIBE)' \
-       -f $(CURDIR)/control-plane/Dockerfile $(CURDIR)/control-plane
+       -f $(CURDIR)/control-plane/Dockerfile $(CURDIR)/control-plane --load
 
 .PHONY: control-plane-dev-skaffold
 # DANGER: this target is experimental and could be modified/removed at any time.

charts/consul/README.md

@@ -103,6 +103,69 @@ Please see the many options supported in the `values.yaml`
 file. These are also fully documented directly on the
 [Consul website](https://www.consul.io/docs/platform/k8s/helm.html).
 
+## API Gateway Probe Configuration
+
+You can configure Kubernetes health probes (liveness, readiness, and startup) for individual API Gateways using annotations. This allows you to customize probe behavior per-Gateway rather than using a class-wide configuration.
+
+### Example Gateway with Custom Probes
+
+```yaml
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: example-gateway
+  namespace: consul
+  annotations:
+    # Configure liveness probe (JSON format)
+    consul.hashicorp.com/liveness-probe: |
+      {
+        "httpGet": {
+          "path": "/ready",
+          "port": 20000
+        },
+        "initialDelaySeconds": 10,
+        "periodSeconds": 10,
+        "timeoutSeconds": 1,
+        "successThreshold": 1,
+        "failureThreshold": 3
+      }
+    # Configure readiness probe (JSON format)
+    consul.hashicorp.com/readiness-probe: |
+      {
+        "httpGet": {
+          "path": "/ready",
+          "port": 20000
+        },
+        "initialDelaySeconds": 5,
+        "periodSeconds": 10
+      }
+    # Configure startup probe (JSON format)
+    consul.hashicorp.com/startup-probe: |
+      {
+        "tcpSocket": {
+          "port": 20000
+        },
+        "periodSeconds": 2,
+        "failureThreshold": 30
+      }
+spec:
+  gatewayClassName: consul
+  listeners:
+  - name: http
+    protocol: HTTP
+    port: 8080
+```
+
+### Supported Probe Annotations
+
+- `consul.hashicorp.com/liveness-probe`: Liveness probe configuration (JSON)
+- `consul.hashicorp.com/readiness-probe`: Readiness probe configuration (JSON)
+- `consul.hashicorp.com/startup-probe`: Startup probe configuration (JSON)
+
+Each annotation accepts a JSON object following the Kubernetes [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#probe-v1-core) specification. Supported probe handlers: `httpGet`, `tcpSocket`, `exec`, `grpc`.
+
+**Note**: Liveness and startup probes must have `successThreshold: 1` per Kubernetes requirements. The controller will automatically normalize this value if a different value is provided.
+
 ## Tutorials
 
 You can find examples and complete tutorials on how to deploy Consul on 

charts/consul/templates/crd-gatewayclassconfigs-v1.yaml

@@ -1,16 +1,19 @@
 {{- if .Values.connectInject.enabled }}
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.14.0
+  name: gatewayclassconfigs.consul.hashicorp.com
   labels:
     app: {{ template "consul.name" . }}
     chart: {{ template "consul.chart" . }}
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
     component: crd
-  name: gatewayclassconfigs.consul.hashicorp.com
 spec:
   group: consul.hashicorp.com
   names:

cli/helm/values.go

@@ -578,13 +578,55 @@ type CopyAnnotations struct {
 }
 
 type ManagedGatewayClass struct {
-	Enabled                     bool            `yaml:"enabled"`
-	NodeSelector                interface{}     `yaml:"nodeSelector"`
-	ServiceType                 string          `yaml:"serviceType"`
-	UseHostPorts                bool            `yaml:"useHostPorts"`
-	CopyAnnotations             CopyAnnotations `yaml:"copyAnnotations"`
-	OpenshiftSCCName            string          `yaml:"openshiftSCCName"`
-	MapPrivilegedContainerPorts int             `yaml:"mapPrivilegedContainerPorts"`
+	Enabled                     bool                      `yaml:"enabled"`
+	NodeSelector                interface{}               `yaml:"nodeSelector"`
+	ServiceType                 string                    `yaml:"serviceType"`
+	UseHostPorts                bool                      `yaml:"useHostPorts"`
+	CopyAnnotations             CopyAnnotations           `yaml:"copyAnnotations"`
+	OpenshiftSCCName            string                    `yaml:"openshiftSCCName"`
+	MapPrivilegedContainerPorts int                       `yaml:"mapPrivilegedContainerPorts"`
+	Probes                      ManagedGatewayClassProbes `yaml:"probes"`
+}
+
+// ProbeHTTPGet models the HTTP GET action of a Kubernetes Probe.
+type ProbeHTTPGet struct {
+	Path   string      `yaml:"path"`
+	Port   interface{} `yaml:"port"` // int or string (named port)
+	Host   string      `yaml:"host"`
+	Scheme string      `yaml:"scheme"`
+	// Headers intentionally omitted for now; can be added if needed.
+}
+
+// ProbeTCPSocket models the TCP socket action of a Kubernetes Probe.
+type ProbeTCPSocket struct {
+	Port interface{} `yaml:"port"` // int or string
+	Host string      `yaml:"host"`
+}
+
+// ProbeExec models the exec action of a Kubernetes Probe.
+type ProbeExec struct {
+	Command []string `yaml:"command"`
+}
+
+// ProbeSpec is a simplified Kubernetes-style Probe configuration allowing http, tcp, or exec.
+// Only one of HTTPGet, TCPSocket, or Exec should be set. Enabled defaults to true if any action is specified.
+type ProbeSpec struct {
+	Enabled             *bool           `yaml:"enabled"`
+	HTTPGet             *ProbeHTTPGet   `yaml:"httpGet"`
+	TCPSocket           *ProbeTCPSocket `yaml:"tcpSocket"`
+	Exec                *ProbeExec      `yaml:"exec"`
+	InitialDelaySeconds int             `yaml:"initialDelaySeconds"`
+	PeriodSeconds       int             `yaml:"periodSeconds"`
+	TimeoutSeconds      int             `yaml:"timeoutSeconds"`
+	SuccessThreshold    int             `yaml:"successThreshold"`
+	FailureThreshold    int             `yaml:"failureThreshold"`
+}
+
+// ManagedGatewayClassProbes groups the three standard Kubernetes probe types applied to gateway pods.
+type ManagedGatewayClassProbes struct {
+	Liveness  ProbeSpec `yaml:"liveness"`
+	Readiness ProbeSpec `yaml:"readiness"`
+	Startup   ProbeSpec `yaml:"startup"`
 }
 
 type Service struct {

control-plane/api-gateway/binding/annotations_test.go

@@ -112,7 +112,7 @@ func TestSerializeGatewayClassConfig_HappyPath(t *testing.T) {
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "my-gw",
 						Annotations: map[string]string{
-							common.AnnotationGatewayClassConfig: `{"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`,
+							common.AnnotationGatewayClassConfig: `{"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"deployment":{},"copyAnnotations":{"service":["service"]},"metrics":{}}`,
 						},
 					},
 					Spec:   gwv1beta1.GatewaySpec{},

control-plane/api-gateway/gatekeeper/deployment.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"strconv"
 
+	"github.com/go-logr/logr"
 	"github.com/google/go-cmp/cmp"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -59,7 +60,7 @@ func (g *Gatekeeper) upsertDeployment(ctx context.Context, gateway gwv1beta1.Gat
 	}
 
 	mutated := deployment.DeepCopy()
-	mutator := newDeploymentMutator(deployment, mutated, existingDeployment, exists, gcc, gateway, g.Client.Scheme())
+	mutator := newDeploymentMutator(deployment, mutated, existingDeployment, exists, gcc, gateway, g.Client.Scheme(), g.Log)
 
 	result, err := controllerutil.CreateOrUpdate(ctx, g.Client, mutated, mutator)
 	if err != nil {
@@ -162,12 +163,35 @@ func (g *Gatekeeper) deployment(gateway gwv1beta1.Gateway, gcc v1alpha1.GatewayC
 	}, nil
 }
 
-func mergeDeployments(gcc v1alpha1.GatewayClassConfig, a, b *appsv1.Deployment) *appsv1.Deployment {
+func mergeDeployments(log logr.Logger, gcc v1alpha1.GatewayClassConfig, gateway gwv1beta1.Gateway, a, b *appsv1.Deployment) *appsv1.Deployment {
 	if !compareDeployments(a, b) {
+		// Replace template
 		b.Spec.Template = a.Spec.Template
 		b.Spec.Replicas = deploymentReplicas(gcc, a.Spec.Replicas)
 	}
 
+	// Apply probes from Gateway annotations if present
+	probes, err := ProbesFromGateway(&gateway)
+	if err != nil {
+		log.Error(err, "failed to parse probe annotations, skipping probe configuration")
+	} else if probes != nil {
+		for i, c := range b.Spec.Template.Spec.Containers {
+			if i > 0 { // only primary container gets managed probes
+				continue
+			}
+			if probes.Liveness != nil {
+				c.LivenessProbe = probes.Liveness.DeepCopy()
+			}
+			if probes.Readiness != nil {
+				c.ReadinessProbe = probes.Readiness.DeepCopy()
+			}
+			if probes.Startup != nil {
+				c.StartupProbe = probes.Startup.DeepCopy()
+			}
+			b.Spec.Template.Spec.Containers[i] = c
+		}
+	}
+
 	return b
 }
 
@@ -209,6 +233,27 @@ func compareDeployments(a, b *appsv1.Deployment) bool {
 				return false
 			}
 		}
+
+		// Compare probe initialDelaySeconds for rollout restart functionality
+		otherContainer := b.Spec.Template.Spec.Containers[i]
+
+		// Compare readiness probe initialDelaySeconds
+		if container.ReadinessProbe != nil && otherContainer.ReadinessProbe != nil {
+			if container.ReadinessProbe.InitialDelaySeconds != otherContainer.ReadinessProbe.InitialDelaySeconds {
+				return false
+			}
+		} else if (container.ReadinessProbe == nil) != (otherContainer.ReadinessProbe == nil) {
+			return false
+		}
+
+		// Compare startup probe initialDelaySeconds
+		if container.StartupProbe != nil && otherContainer.StartupProbe != nil {
+			if container.StartupProbe.InitialDelaySeconds != otherContainer.StartupProbe.InitialDelaySeconds {
+				return false
+			}
+		} else if (container.StartupProbe == nil) != (otherContainer.StartupProbe == nil) {
+			return false
+		}
 	}
 
 	if b.Spec.Replicas == nil && a.Spec.Replicas == nil {
@@ -234,9 +279,9 @@ func mergeAnnotation(b *appsv1.Deployment, annotations map[string]string) {
 
 }
 
-func newDeploymentMutator(deployment, mutated, existingDeployment *appsv1.Deployment, deploymentExists bool, gcc v1alpha1.GatewayClassConfig, gateway gwv1beta1.Gateway, scheme *runtime.Scheme) resourceMutator {
+func newDeploymentMutator(deployment, mutated, existingDeployment *appsv1.Deployment, deploymentExists bool, gcc v1alpha1.GatewayClassConfig, gateway gwv1beta1.Gateway, scheme *runtime.Scheme, log logr.Logger) resourceMutator {
 	return func() error {
-		mutated = mergeDeployments(gcc, deployment, mutated)
+		mutated = mergeDeployments(log, gcc, gateway, deployment, mutated)
 		if deploymentExists {
 			mergeAnnotation(mutated, existingDeployment.Spec.Template.Annotations)
 		}

control-plane/api-gateway/gatekeeper/deployment_test.go

@@ -6,11 +6,15 @@ package gatekeeper
 import (
 	"testing"
 
+	"github.com/go-logr/logr"
 	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
 	"github.com/hashicorp/consul-k8s/control-plane/api-gateway/common"
+	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
 )
 
 func Test_compareDeployments(t *testing.T) {
@@ -217,3 +221,74 @@ func Test_compareDeployments(t *testing.T) {
 		})
 	}
 }
+
+func TestMergeDeployments_ProbePropagation(t *testing.T) {
+	t.Parallel()
+
+	log := logr.Discard()
+
+	gcc := v1alpha1.GatewayClassConfig{}
+
+	gateway := gwv1beta1.Gateway{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-gateway",
+			Annotations: map[string]string{
+				AnnotationLivenessProbe: `{
+					"httpGet": {
+						"path": "/health",
+						"port": 8080
+					}
+				}`,
+			},
+		},
+	}
+
+	deployment := &appsv1.Deployment{
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "consul-dataplane",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	merged := mergeDeployments(log, gcc, gateway, deployment, &appsv1.Deployment{})
+	assert.NotNil(t, merged)
+
+	// Verify probe was applied to the container
+	assert.Len(t, merged.Spec.Template.Spec.Containers, 1)
+	container := merged.Spec.Template.Spec.Containers[0]
+	assert.NotNil(t, container.LivenessProbe)
+	assert.NotNil(t, container.LivenessProbe.HTTPGet)
+	assert.Equal(t, "/health", container.LivenessProbe.HTTPGet.Path)
+	assert.Equal(t, int32(8080), container.LivenessProbe.HTTPGet.Port.IntVal)
+
+	// Now update Gateway with different probe (TCPSocket instead of HTTPGet)
+	gatewayUpdated := gwv1beta1.Gateway{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-gateway",
+			Annotations: map[string]string{
+				AnnotationLivenessProbe: `{
+					"tcpSocket": {
+						"port": 9090
+					}
+				}`,
+			},
+		},
+	}
+
+	mergedUpdated := mergeDeployments(log, gcc, gatewayUpdated, merged, &appsv1.Deployment{})
+	assert.NotNil(t, mergedUpdated)
+
+	// Verify the probe handler was replaced (TCPSocket instead of HTTPGet)
+	containerUpdated := mergedUpdated.Spec.Template.Spec.Containers[0]
+	assert.NotNil(t, containerUpdated.LivenessProbe)
+	assert.NotNil(t, containerUpdated.LivenessProbe.TCPSocket)
+	assert.Nil(t, containerUpdated.LivenessProbe.HTTPGet)
+	assert.Equal(t, int32(9090), containerUpdated.LivenessProbe.TCPSocket.Port.IntVal)
+}