Skip to content

Commit a9989cd

Browse files
zac-nixonzac-nixon
committed
e2e test for tcp_udp
1 parent fcb9fb2 commit a9989cd

File tree

5 files changed

+242
-13
lines changed

5 files changed

+242
-13
lines changed

‎controllers/gateway/targetgroup_configuration_controller.go‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"github.com/go-logr/logr"
7+
corev1 "k8s.io/api/core/v1"
78
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
89
"k8s.io/apimachinery/pkg/types"
910
"k8s.io/client-go/kubernetes"
@@ -115,11 +116,25 @@ func (r *targetgroupConfigurationReconciler) handleDelete(tgConf *elbv2gw.Target
115116
Name: tgConf.Spec.TargetReference.Name,
116117
}
117118

118-
eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways)
119+
svc := &corev1.Service{}
120+
err := r.k8sClient.Get(context.Background(), svcReference, svc)
119121

120-
// if the targetgroup configuration is still in use, we should not delete it
121-
if !eligibleForRemoval {
122-
return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf))
122+
referenceCheckNeeded := true
123+
if err != nil {
124+
notFoundErr := client.IgnoreNotFound(err)
125+
if notFoundErr != nil {
126+
return notFoundErr
127+
}
128+
referenceCheckNeeded = false
129+
}
130+
131+
if referenceCheckNeeded {
132+
eligibleForRemoval := r.serviceReferenceCounter.IsEligibleForRemoval(svcReference, allGateways)
133+
134+
// if the targetgroup configuration is still in use, we should not delete it
135+
if !eligibleForRemoval {
136+
return fmt.Errorf("targetgroup configuration [%+v] is still in use", k8s.NamespacedName(tgConf))
137+
}
123138
}
124139
return r.finalizerManager.RemoveFinalizers(context.Background(), tgConf, shared_constants.TargetGroupConfigurationFinalizer)
125140
}

‎test/e2e/gateway/nlb_instance_target_test.go‎

Lines changed: 92 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,16 @@ package gateway
33
import (
44
"context"
55
"fmt"
6-
"strconv"
7-
"strings"
8-
"time"
9-
106
awssdk "github.com/aws/aws-sdk-go-v2/aws"
117
. "github.com/onsi/ginkgo/v2"
128
. "github.com/onsi/gomega"
139
elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
1410
"sigs.k8s.io/aws-load-balancer-controller/test/framework/http"
1511
"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
1612
"sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier"
13+
"strconv"
14+
"strings"
15+
"time"
1716
)
1817

1918
var _ = Describe("test nlb gateway using instance targets reconciled by the aws load balancer controller", func() {
@@ -34,8 +33,11 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws
3433
})
3534
AfterEach(func() {
3635
stack.Cleanup(ctx, tf)
37-
auxiliaryStack.Cleanup(ctx, tf)
36+
if auxiliaryStack != nil {
37+
auxiliaryStack.Cleanup(ctx, tf)
38+
}
3839
})
40+
3941
Context(fmt.Sprintf("with NLB instance target configuration, using readiness gates %+v", false), func() {
4042
BeforeEach(func() {})
4143
It("should provision internet-facing load balancer resources", func() {
@@ -567,5 +569,90 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws
567569
validateL4RouteStatusNotPermitted(tf, stack, hasTLS)
568570
})
569571
})
572+
Context(fmt.Sprintf("with NLB instance target using TCP_UDP listener"), func() {
573+
BeforeEach(func() {})
574+
It("should provision internet-facing load balancer resources", func() {
575+
interf := elbv2gw.LoadBalancerSchemeInternetFacing
576+
lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{
577+
Scheme: &interf,
578+
}
579+
580+
instanceTargetType := elbv2gw.TargetTypeInstance
581+
tgSpec := elbv2gw.TargetGroupConfigurationSpec{
582+
DefaultConfiguration: elbv2gw.TargetGroupProps{
583+
TargetType: &instanceTargetType,
584+
},
585+
}
586+
By("deploying stack", func() {
587+
err := stack.DeployTCP_UDP(ctx, tf, lbcSpec, tgSpec, false)
588+
Expect(err).NotTo(HaveOccurred())
589+
})
590+
591+
By("checking gateway status for lb dns name", func() {
592+
dnsName = stack.GetLoadBalancerIngressHostName()
593+
Expect(dnsName).ToNot(BeEmpty())
594+
})
595+
596+
By("querying AWS loadbalancer from the dns name", func() {
597+
var err error
598+
lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
599+
Expect(err).NotTo(HaveOccurred())
600+
Expect(lbARN).ToNot(BeEmpty())
601+
})
602+
603+
By("verifying AWS loadbalancer resources", func() {
604+
nodeList, err := stack.GetWorkerNodes(ctx, tf)
605+
Expect(err).ToNot(HaveOccurred())
606+
expectedTargetGroups := []verifier.ExpectedTargetGroup{
607+
{
608+
Protocol: "TCP_UDP",
609+
Port: stack.nlbResourceStack.commonStack.svcs[0].Spec.Ports[0].NodePort,
610+
NumTargets: len(nodeList),
611+
TargetType: "instance",
612+
TargetGroupHC: &verifier.TargetGroupHC{
613+
Protocol: "TCP",
614+
Port: "traffic-port",
615+
Interval: 15,
616+
Timeout: 5,
617+
HealthyThreshold: 3,
618+
UnhealthyThreshold: 3,
619+
},
620+
},
621+
}
622+
623+
listenerPortMap := map[string]string{
624+
"80": "TCP_UDP",
625+
}
626+
627+
err = verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{
628+
Type: "network",
629+
Scheme: "internet-facing",
630+
Listeners: listenerPortMap,
631+
TargetGroups: expectedTargetGroups,
632+
})
633+
Expect(err).NotTo(HaveOccurred())
634+
})
635+
By("waiting for target group targets to be healthy", func() {
636+
nodeList, err := stack.GetWorkerNodes(ctx, tf)
637+
Expect(err).ToNot(HaveOccurred())
638+
err = verifier.WaitUntilTargetsAreHealthy(ctx, tf, lbARN, len(nodeList))
639+
Expect(err).NotTo(HaveOccurred())
640+
})
641+
By("waiting until DNS name is available", func() {
642+
err := utils.WaitUntilDNSNameAvailable(ctx, dnsName)
643+
Expect(err).NotTo(HaveOccurred())
644+
})
645+
By("sending http request to the lb", func() {
646+
url := fmt.Sprintf("http://%v/any-path", dnsName)
647+
err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200))
648+
Expect(err).NotTo(HaveOccurred())
649+
})
650+
By("sending udp request to the lb", func() {
651+
endpoint := fmt.Sprintf("%v:80", dnsName)
652+
err := tf.UDPVerifier.VerifyUDP(endpoint)
653+
Expect(err).NotTo(HaveOccurred())
654+
})
655+
})
656+
})
570657
})
571658
})

‎test/e2e/gateway/nlb_ip_target_test.go‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,12 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b
3232
})
3333
AfterEach(func() {
3434
stack.Cleanup(ctx, tf)
35-
auxiliaryStack.Cleanup(ctx, tf)
35+
if auxiliaryStack != nil {
36+
auxiliaryStack.Cleanup(ctx, tf)
37+
}
3638
})
3739
for _, readinessGateEnabled := range []bool{true, false} {
40+
3841
Context(fmt.Sprintf("with NLB ip target configuration, using readiness gates %+v", readinessGateEnabled), func() {
3942
It("should provision internet-facing load balancer resources", func() {
4043
interf := elbv2gw.LoadBalancerSchemeInternetFacing
@@ -282,6 +285,7 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b
282285
})
283286
})
284287
})
288+
285289
Context(fmt.Sprintf("with NLB ip target configuration, using no SG, using readiness gates %+v", readinessGateEnabled), func() {
286290
It("should provision internet-facing load balancer resources", func() {
287291
interf := elbv2gw.LoadBalancerSchemeInternetFacing
@@ -529,5 +533,91 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b
529533
})
530534
})
531535
})
536+
537+
Context(fmt.Sprintf("with TCP_UDP listener, using readiness gates %+v", readinessGateEnabled), func() {
538+
It("should provision internet-facing load balancer resources", func() {
539+
interf := elbv2gw.LoadBalancerSchemeInternetFacing
540+
lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{
541+
Scheme: &interf,
542+
}
543+
544+
ipTargetType := elbv2gw.TargetTypeIP
545+
tgSpec := elbv2gw.TargetGroupConfigurationSpec{
546+
DefaultConfiguration: elbv2gw.TargetGroupProps{
547+
TargetType: &ipTargetType,
548+
},
549+
}
550+
551+
By("deploying stack", func() {
552+
err := stack.DeployTCP_UDP(ctx, tf, lbcSpec, tgSpec, false)
553+
Expect(err).NotTo(HaveOccurred())
554+
})
555+
556+
By("checking gateway status for lb dns name", func() {
557+
dnsName = stack.GetLoadBalancerIngressHostName()
558+
Expect(dnsName).ToNot(BeEmpty())
559+
})
560+
561+
By("querying AWS loadbalancer from the dns name", func() {
562+
var err error
563+
lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
564+
Expect(err).NotTo(HaveOccurred())
565+
Expect(lbARN).ToNot(BeEmpty())
566+
})
567+
568+
targetNumber := int(*stack.nlbResourceStack.commonStack.dps[0].Spec.Replicas)
569+
570+
expectedTargetGroups := []verifier.ExpectedTargetGroup{
571+
{
572+
Protocol: "TCP_UDP",
573+
Port: 8080,
574+
NumTargets: targetNumber,
575+
TargetType: "ip",
576+
TargetGroupHC: &verifier.TargetGroupHC{
577+
Protocol: "TCP",
578+
Port: "traffic-port",
579+
Interval: 15,
580+
Timeout: 5,
581+
HealthyThreshold: 3,
582+
UnhealthyThreshold: 3,
583+
},
584+
},
585+
}
586+
587+
listenerPortMap := map[string]string{
588+
"80": "TCP_UDP",
589+
}
590+
591+
fmt.Println("------VALIDATE NOW!--------")
592+
time.Sleep(10 * time.Minute)
593+
By("verifying AWS loadbalancer resources", func() {
594+
err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{
595+
Type: "network",
596+
Scheme: "internet-facing",
597+
Listeners: listenerPortMap,
598+
TargetGroups: expectedTargetGroups,
599+
})
600+
Expect(err).NotTo(HaveOccurred())
601+
})
602+
By("waiting for target group targets to be healthy", func() {
603+
err := verifier.WaitUntilTargetsAreHealthy(ctx, tf, lbARN, targetNumber)
604+
Expect(err).NotTo(HaveOccurred())
605+
})
606+
By("waiting until DNS name is available", func() {
607+
err := utils.WaitUntilDNSNameAvailable(ctx, dnsName)
608+
Expect(err).NotTo(HaveOccurred())
609+
})
610+
By("sending http request to the lb", func() {
611+
url := fmt.Sprintf("http://%v/any-path", dnsName)
612+
err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200))
613+
Expect(err).NotTo(HaveOccurred())
614+
})
615+
By("sending udp request to the lb", func() {
616+
endpoint := fmt.Sprintf("%v:80", dnsName)
617+
err := tf.UDPVerifier.VerifyUDP(endpoint)
618+
Expect(err).NotTo(HaveOccurred())
619+
})
620+
})
621+
})
532622
}
533623
})

‎test/e2e/gateway/nlb_test_helper.go‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,49 @@ func (s *NLBTestStack) Deploy(ctx context.Context, f *framework.Framework, auxil
8585
lbc := buildLoadBalancerConfig(lbConfSpec)
8686
tgcTCP := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svcTCP)
8787
tgcUDP := buildTargetGroupConfig(udpDefaultTgConfigName, tgConfSpec, svcUDP)
88-
udpr := buildUDPRoute()
88+
udpr := buildUDPRoute("port8080")
8989

9090
s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{dpTCP, dpUDP}, []*corev1.Service{svcTCP, svcUDP}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgcTCP, tgcUDP}, tcprs, []*gwalpha2.UDPRoute{udpr}, nil, "nlb-gateway-e2e", readinessGateEnabled)
9191

9292
return s.nlbResourceStack.Deploy(ctx, f)
9393
}
9494

95+
func (s *NLBTestStack) DeployTCP_UDP(ctx context.Context, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec, readinessGateEnabled bool) error {
96+
dpUDP := buildUDPDeploymentSpec()
97+
svcUDP := buildUDPServiceSpec()
98+
gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb")
99+
100+
if f.Options.IPFamily == framework.IPv6 {
101+
v6 := elbv2gw.LoadBalancerIpAddressTypeDualstack
102+
lbConfSpec.IpAddressType = &v6
103+
}
104+
105+
listeners := []gwv1.Listener{
106+
{
107+
Name: "port80tcp",
108+
Port: 80,
109+
Protocol: gwv1.TCPProtocolType,
110+
},
111+
{
112+
Name: "port80udp",
113+
Port: 80,
114+
Protocol: gwv1.UDPProtocolType,
115+
},
116+
}
117+
118+
tcprs := []*gwalpha2.TCPRoute{}
119+
120+
gw := buildBasicGatewaySpec(gwc, listeners)
121+
122+
lbc := buildLoadBalancerConfig(lbConfSpec)
123+
tgcUDP := buildTargetGroupConfig(udpDefaultTgConfigName, tgConfSpec, svcUDP)
124+
udpr := buildUDPRoute("port80udp")
125+
126+
s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{dpUDP}, []*corev1.Service{svcUDP}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgcUDP}, tcprs, []*gwalpha2.UDPRoute{udpr}, nil, "nlb-gateway-e2e", readinessGateEnabled)
127+
128+
return s.nlbResourceStack.Deploy(ctx, f)
129+
}
130+
95131
func (s *NLBTestStack) DeployFrontendNLB(ctx context.Context, albStack ALBTestStack, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, hasTLS bool, readinessGateEnabled bool) error {
96132
gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb")
97133

@@ -285,6 +321,7 @@ func validateL4RouteStatusNotPermitted(tf *framework.Framework, stack NLBTestSta
285321
},
286322
},
287323
}
324+
288325
validateRouteStatus(tf, stack.nlbResourceStack.tcprs, tcpRouteStatusConverter, tcpValidationInfo)
289326
validateRouteStatus(tf, stack.nlbResourceStack.udprs, udpRouteStatusConverter, udpValidationInfo)
290327
}

‎test/e2e/gateway/shared_resource_definitions.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ func buildFENLBTCPRoute(albGatewayName, albNamespace string, port gwalpha2.PortN
348348
return tcpr
349349
}
350350

351-
func buildUDPRoute() *gwalpha2.UDPRoute {
351+
func buildUDPRoute(sectionName string) *gwalpha2.UDPRoute {
352352
port := gwalpha2.PortNumber(8080)
353353
udpr := &gwalpha2.UDPRoute{
354354
ObjectMeta: metav1.ObjectMeta{
@@ -359,7 +359,7 @@ func buildUDPRoute() *gwalpha2.UDPRoute {
359359
ParentRefs: []gwv1.ParentReference{
360360
{
361361
Name: defaultName,
362-
SectionName: (*gwv1.SectionName)(awssdk.String("port8080")),
362+
SectionName: (*gwv1.SectionName)(awssdk.String(sectionName)),
363363
},
364364
},
365365
},

0 commit comments

Comments
 (0)