Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apis/gateway/v1beta1/loadbalancerconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ type WAFv2Configuration struct {
ACL string `json:"webACL"`
}

// +kubebuilder:validation:Pattern="^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{4}|[1-9]\\d{0,3})?$"
// +kubebuilder:validation:Pattern="^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{4}|[1-9]\\d{0,3})?$"
type ProtocolPort string
type ListenerConfiguration struct {
// protocolPort is identifier for the listener on load balancer. It should be of the form PROTOCOL:PORT
Expand Down
2 changes: 1 addition & 1 deletion config/crd/gateway/gateway-crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ spec:
protocolPort:
description: protocolPort is identifier for the listener on
load balancer. It should be of the form PROTOCOL:PORT
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
type: string
sslPolicy:
description: sslPolicy is the security policy that defines which
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ spec:
protocolPort:
description: protocolPort is identifier for the listener on
load balancer. It should be of the form PROTOCOL:PORT
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
type: string
sslPolicy:
description: sslPolicy is the security policy that defines which
Expand Down
23 changes: 19 additions & 4 deletions controllers/gateway/targetgroup_configuration_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -115,11 +116,25 @@ func (r *targetgroupConfigurationReconciler) handleDelete(tgConf *elbv2gw.Target
Name: tgConf.Spec.TargetReference.Name,
}

eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways)
svc := &corev1.Service{}
err := r.k8sClient.Get(context.Background(), svcReference, svc)

// if the targetgroup configuration is still in use, we should not delete it
if !eligibleForRemoval {
return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf))
referenceCheckNeeded := true
if err != nil {
notFoundErr := client.IgnoreNotFound(err)
if notFoundErr != nil {
return notFoundErr
}
referenceCheckNeeded = false
}

if referenceCheckNeeded {
eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways)

// if the targetgroup configuration is still in use, we should not delete it
if !eligibleForRemoval {
return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf))
}
}
return r.finalizerManager.RemoveFinalizers(context.Background(), tgConf, shared_constants.TargetGroupConfigurationFinalizer)
}
Expand Down
89 changes: 89 additions & 0 deletions docs/guide/gateway/l4gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,95 @@ spec:
* **L4 Listener Materialization:** The controller processes the `my-tcp-app-route` resource. Given that the `TCPRoute` validly references the `my-tcp-gateway` and its `tcp-app` listener, an **NLB Listener** is materialized on the provisioned NLB. This listener will be configured for `TCP` protocol on `port 8080`, as specified in the `Gateway`'s listener definition. A default forward action is subsequently configured on the NLB Listener, directing all incoming traffic on `port 8080` to the newly created Target Group for service `my-tcp-service` in `backendRefs` section of `my-tcp-app-route`.
* **Target Group Creation:** An **AWS Target Group** is created for the Kubernetes Service `my-tcp-service` with default configuration. The cluster nodes are then registered as targets within this new Target Group.


### Combined Protocols
AWS NLB supports combining TCP and UDP on the same listener; the protocol is called TCP_UDP. This powerful
paradigm allows the load balancer to serve different protocols for different applications on the same listener port.
The LBC implements this protocol merging capability.

#### Combined protocol quirks

AWS NLB assumes that in a combined protocol set up,
all targets are able to serve both protocols. To prevent configuration duplication, we follow this same pattern for constructing
the combined protocol listener. TCP_UDP listeners are able to attach routes of type TCP and UDP, each route attached
generates a TCP_UDP target group.


#### Combined protocol examples

```yaml
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: my-tcp-udp-gateway
namespace: tcp-udp
spec:
gatewayClassName: aws-nlb-gateway-class
listeners:
- allowedRoutes:
namespaces:
from: Same
name: tcp-app
port: 80
protocol: TCP
- allowedRoutes:
namespaces:
from: Same
name: udp-app
port: 80
protocol: UDP
---
apiVersion: gateway.networking.k8s.io/v1alpha2
kind: UDPRoute
metadata:
name: my-udp-app-route
namespace: tcp-udp
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: my-tcp-udp-gateway
sectionName: udp-app
rules:
- backendRefs:
- group: ""
kind: Service
name: udpechoserver
port: 8080
weight: 1
```

To customize the target group created, it's no different from a single protocol
```yaml
apiVersion: gateway.k8s.aws/v1beta1
kind: TargetGroupConfiguration
metadata:
name: example-tg-config
namespace: tcp-udp
spec:
defaultConfiguration:
targetType: ip
targetReference:
group: ""
kind: Service
name: udpechoserver
```

To customize the listener:
```yaml
apiVersion: gateway.k8s.aws/v1beta1
kind: LoadBalancerConfiguration
metadata:
name: nlb-lb-config
namespace: tcp-udp
spec:
listenerConfigurations:
- protocolPort: TCP_UDP:80
listenerAttributes:
- key: tcp.idle_timeout.seconds
value: "60"
```

### L4 Gateway API Limitations for NLBs
The LBC implementation of the Gateway API for L4 routes, which provisions NLB, introduces specific constraints to align with NLB capabilities. These limitations are enforced during the reconciliation process and are critical for successful L4 traffic management.

Expand Down
2 changes: 1 addition & 1 deletion helm/aws-load-balancer-controller/crds/gateway-crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ spec:
protocolPort:
description: protocolPort is identifier for the listener on
load balancer. It should be of the form PROTOCOL:PORT
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
pattern: ^(HTTP|HTTPS|TLS|TCP|UDP|TCP_UDP)?:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3})?$
type: string
sslPolicy:
description: sslPolicy is the security policy that defines which
Expand Down
11 changes: 6 additions & 5 deletions pkg/deploy/elbv2/listener_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@ var alpnNone = []string{
}

var PROTOCOLS_SUPPORTING_LISTENER_ATTRIBUTES = map[elbv2model.Protocol]bool{
elbv2model.ProtocolHTTP: true,
elbv2model.ProtocolHTTPS: true,
elbv2model.ProtocolTCP: true,
elbv2model.ProtocolUDP: false,
elbv2model.ProtocolTLS: false,
elbv2model.ProtocolHTTP: true,
elbv2model.ProtocolHTTPS: true,
elbv2model.ProtocolTCP: true,
elbv2model.ProtocolUDP: false,
elbv2model.ProtocolTLS: false,
elbv2model.ProtocolTCP_UDP: true,
}

// ListenerManager is responsible for create/update/delete Listener resources.
Expand Down
2 changes: 1 addition & 1 deletion pkg/gateway/model/mock_tg_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func (m *mockTargetGroupBuilder) getLocalFrontendNlbData() map[string]*elbv2mode
}

func (m *mockTargetGroupBuilder) buildTargetGroup(stack core.Stack,
gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) {
gw *gwv1.Gateway, listenerPort int32, listenerProtocol elbv2model.Protocol, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) {
var tg *elbv2model.TargetGroup

if len(m.tgs) > 0 {
Expand Down
79 changes: 56 additions & 23 deletions pkg/gateway/model/model_build_listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac
portsWithRoutes := sets.Int32KeySet(routes)
// Materialise the listener only if listener has associated routes
if len(gwLsPorts.Intersection(portsWithRoutes).List()) != 0 {
lbLsCfgs := mapLoadBalancerListenerConfigsByPort(lbCfg, gw.Spec.Listeners)
lbLsCfgs := mapLoadBalancerListenerConfigsByPort(lbCfg, gwLsCfgs)
for _, port := range gwLsPorts.Intersection(portsWithRoutes).List() {
ls, err := l.buildListener(ctx, stack, lb, gw, port, routes[port], lbCfg, gwLsCfgs[port], lbLsCfgs[port])
if err != nil {
Expand All @@ -83,7 +83,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac
return secrets, nil
}

func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, routes []routeutils.RouteDescriptor, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.Listener, error) {
func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, routes []routeutils.RouteDescriptor, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.Listener, error) {
var listenerSpec *elbv2model.ListenerSpec

var err error
Expand All @@ -104,7 +104,7 @@ func (l listenerBuilderImpl) buildListener(ctx context.Context, stack core.Stack
return elbv2model.NewListener(stack, lsResID, *listenerSpec), nil
}

func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
tags, err := l.buildListenerTags(lbCfg)
if err != nil {
return &elbv2model.ListenerSpec{}, err
Expand Down Expand Up @@ -133,7 +133,7 @@ func (l listenerBuilderImpl) buildListenerSpec(ctx context.Context, lb *elbv2mod
return listenerSpec, nil
}

func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
listenerSpec, err := l.buildListenerSpec(ctx, lb, gw, port, lbCfg, gwLsCfg, lbLsCfg)
if err != nil {
return &elbv2model.ListenerSpec{}, err
Expand All @@ -147,7 +147,7 @@ func (l listenerBuilderImpl) buildL7ListenerSpec(ctx context.Context, lb *elbv2m
return listenerSpec, nil
}

func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, routes []routeutils.RouteDescriptor, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core.Stack, lb *elbv2model.LoadBalancer, gw *gwv1.Gateway, lbCfg elbv2gw.LoadBalancerConfiguration, port int32, routes []routeutils.RouteDescriptor, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.ListenerSpec, error) {
listenerSpec, err := l.buildListenerSpec(ctx, lb, gw, port, lbCfg, gwLsCfg, lbLsCfg)
if err != nil {
return &elbv2model.ListenerSpec{}, err
Expand Down Expand Up @@ -177,7 +177,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core
return nil, nil
}

arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, lb.Spec.IPAddressType, routeDescriptor, backend)
arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, listenerSpec.Protocol, lb.Spec.IPAddressType, routeDescriptor, backend)
if tgErr != nil {
return &elbv2model.ListenerSpec{}, tgErr
}
Expand Down Expand Up @@ -222,7 +222,7 @@ func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core.
}
targetGroupTuples := make([]elbv2model.TargetGroupTuple, 0, len(rule.GetBackends()))
for _, backend := range rule.GetBackends() {
arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ipAddressType, route, backend)
arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ls.Spec.Protocol, ipAddressType, route, backend)
if tgErr != nil {
return nil, tgErr
}
Expand Down Expand Up @@ -313,7 +313,7 @@ func buildListenerAttributes(lsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model
return attributes, nil
}

func (l listenerBuilderImpl) buildCertificates(ctx context.Context, gw *gwv1.Gateway, port int32, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model.Certificate, error) {
func (l listenerBuilderImpl) buildCertificates(ctx context.Context, gw *gwv1.Gateway, port int32, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) ([]elbv2model.Certificate, error) {
if !isSecureProtocol(gwLsCfg.protocol) {
return []elbv2model.Certificate{}, nil
}
Expand Down Expand Up @@ -408,7 +408,7 @@ func buildL4ListenerDefaultActions(arn core.StringToken) []elbv2model.Action {
}
}

func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Context, gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.MutualAuthenticationAttributes, error) {
func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Context, gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*elbv2model.MutualAuthenticationAttributes, error) {
// Skip mTLS configuration for non-secure protocols
if !isSecureProtocol(gwLsCfg.protocol) || lbLsCfg == nil || lbLsCfg.MutualAuthentication == nil {
return nil, nil
Expand Down Expand Up @@ -453,7 +453,7 @@ func (l listenerBuilderImpl) buildMutualAuthenticationAttributes(ctx context.Con
}, nil
}

func (l listenerBuilderImpl) buildSSLPolicy(gwLsCfg *gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*string, error) {
func (l listenerBuilderImpl) buildSSLPolicy(gwLsCfg gwListenerConfig, lbLsCfg *elbv2gw.ListenerConfiguration) (*string, error) {
if !isSecureProtocol(gwLsCfg.protocol) {
return nil, nil
}
Expand Down Expand Up @@ -488,17 +488,34 @@ func buildListenerALPNPolicy(listenerProtocol elbv2model.Protocol, lbLsCfg *elbv
}

// mapGatewayListenerConfigsByPort creates a mapping of ports to listener configurations from the Gateway listeners.
func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeutils.RouteDescriptor) (map[int32]*gwListenerConfig, error) {
gwListenerConfigs := make(map[int32]*gwListenerConfig)
func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeutils.RouteDescriptor) (map[int32]gwListenerConfig, error) {
gwListenerConfigs := make(map[int32]gwListenerConfig)
for _, listener := range gw.Spec.Listeners {
port := int32(listener.Port)
protocol := listener.Protocol
if gwListenerConfigs[port] != nil && string(gwListenerConfigs[port].protocol) != string(protocol) {
return nil, fmt.Errorf("invalid listeners on gateway, listeners with same ports cannot have different protocols")
protocol := elbv2model.Protocol(listener.Protocol)

_, hasPort := gwListenerConfigs[port]
if !hasPort {
gwListenerConfigs[port] = gwListenerConfig{
protocol: protocol,
hostnames: sets.New[string](),
}
}
if gwListenerConfigs[port] == nil {
gwListenerConfigs[port] = &gwListenerConfig{
protocol: elbv2model.Protocol(protocol),

if hasPort && gwListenerConfigs[port].protocol != protocol {
// Special case TCP_UDP (or TCP_QUIC)

mergedValue, mergeErr := mergeProtocols(gwListenerConfigs[port].protocol, protocol)

if mergeErr != nil {
return nil, fmt.Errorf("invalid listeners on gateway, listeners with same ports cannot have different protocols")
}

// TODO this only works for TCP, UDP route merging.
// If we need to support TLS merging, then this will need
// to be updated.
gwListenerConfigs[port] = gwListenerConfig{
protocol: mergedValue,
hostnames: sets.New[string](),
}
}
Expand Down Expand Up @@ -532,11 +549,11 @@ func mapGatewayListenerConfigsByPort(gw *gwv1.Gateway, routes map[int32][]routeu

// mapLoadBalancerListenerConfigsByPort creates a mapping of ports to their corresponding
// listener configurations from the LoadBalancer configuration.
func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguration, gatewayListeners []gwv1.Listener) map[int32]*elbv2gw.ListenerConfiguration {
func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguration, gatewayListeners map[int32]gwListenerConfig) map[int32]*elbv2gw.ListenerConfiguration {
configuredListeners := sets.NewString()

for _, configuredListener := range gatewayListeners {
configuredListeners.Insert(generateListenerPortKey(configuredListener))
for port, configuredListener := range gatewayListeners {
configuredListeners.Insert(generateListenerPortKey(port, configuredListener))
}

lbLsCfgs := make(map[int32]*elbv2gw.ListenerConfiguration)
Expand All @@ -554,8 +571,8 @@ func mapLoadBalancerListenerConfigsByPort(lbCfg elbv2gw.LoadBalancerConfiguratio
return lbLsCfgs
}

func generateListenerPortKey(listener gwv1.Listener) string {
return fmt.Sprintf("%s:%d", strings.ToLower(string(listener.Protocol)), listener.Port)
func generateListenerPortKey(port int32, listener gwListenerConfig) string {
return fmt.Sprintf("%s:%d", strings.ToLower(string(listener.protocol)), port)
}

func newListenerBuilder(loadBalancerType elbv2model.LoadBalancerType, tgBuilder targetGroupBuilder, tagHelper tagHelper, clusterName string, defaultSSLPolicy string, elbv2Client services.ELBV2, acmClient services.ACM, k8sClient client.Client, allowedCAARNs []string, secretsManager k8s.SecretsManager, logger logr.Logger) listenerBuilder {
Expand Down Expand Up @@ -599,3 +616,19 @@ func getRoutingAction(config *elbv2gw.ListenerRuleConfiguration) *elbv2gw.Action
}
return nil
}

func mergeProtocols(storedProtocol, proposedProtocol elbv2model.Protocol) (elbv2model.Protocol, error) {
if storedProtocol == elbv2model.ProtocolTCP_UDP && (proposedProtocol == elbv2model.ProtocolTCP || proposedProtocol == elbv2model.ProtocolUDP) {
return elbv2model.ProtocolTCP_UDP, nil
}

if storedProtocol == elbv2model.ProtocolTCP && proposedProtocol == elbv2model.ProtocolUDP {
return elbv2model.ProtocolTCP_UDP, nil
}

if storedProtocol == elbv2model.ProtocolUDP && proposedProtocol == elbv2model.ProtocolTCP {
return elbv2model.ProtocolTCP_UDP, nil
}

return elbv2model.ProtocolHTTP, errors.New("unsupported merge")
}
Loading
Loading