Skip to content

Commit 0da8259

Browse files
sjbermansjberman
committed
CP/DP split: update/delete user secrets (#3193)
Problem: When a user updates or deletes their docker registry or NGINX Plus secrets, those changes need to be propagated to all duplicate secrets that we've provisioned for the Gateway resources. Solution: If updated, update the provisioned secret. If deleted, delete the provisioned secret.
1 parent 6cc3add commit 0da8259

File tree

11 files changed

+669
-25
lines changed

11 files changed

+669
-25
lines changed

‎internal/framework/controller/predicate/annotation.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,24 @@ type AnnotationPredicate struct {
1919
}
2020

2121
// Create filters CreateEvents based on the Annotation.
22-
func (cp AnnotationPredicate) Create(e event.CreateEvent) bool {
22+
func (ap AnnotationPredicate) Create(e event.CreateEvent) bool {
2323
if e.Object == nil {
2424
return false
2525
}
2626

27-
_, ok := e.Object.GetAnnotations()[cp.Annotation]
27+
_, ok := e.Object.GetAnnotations()[ap.Annotation]
2828
return ok
2929
}
3030

3131
// Update filters UpdateEvents based on the Annotation.
32-
func (cp AnnotationPredicate) Update(e event.UpdateEvent) bool {
32+
func (ap AnnotationPredicate) Update(e event.UpdateEvent) bool {
3333
if e.ObjectOld == nil || e.ObjectNew == nil {
3434
// this case should not happen
3535
return false
3636
}
3737

38-
oldAnnotationVal := e.ObjectOld.GetAnnotations()[cp.Annotation]
39-
newAnnotationVal := e.ObjectNew.GetAnnotations()[cp.Annotation]
38+
oldAnnotationVal := e.ObjectOld.GetAnnotations()[ap.Annotation]
39+
newAnnotationVal := e.ObjectNew.GetAnnotations()[ap.Annotation]
4040

4141
return oldAnnotationVal != newAnnotationVal
4242
}
@@ -52,7 +52,7 @@ type RestartDeploymentAnnotationPredicate struct {
5252
}
5353

5454
// Update filters UpdateEvents based on if the annotation is present or changed.
55-
func (cp RestartDeploymentAnnotationPredicate) Update(e event.UpdateEvent) bool {
55+
func (RestartDeploymentAnnotationPredicate) Update(e event.UpdateEvent) bool {
5656
if e.ObjectOld == nil || e.ObjectNew == nil {
5757
// this case should not happen
5858
return false

‎internal/framework/controller/predicate/secret.go

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package predicate
2+
3+
import (
4+
"slices"
5+
6+
corev1 "k8s.io/api/core/v1"
7+
"sigs.k8s.io/controller-runtime/pkg/event"
8+
"sigs.k8s.io/controller-runtime/pkg/predicate"
9+
)
10+
11+
// SecretNamePredicate implements a predicate function that returns true if the Secret matches the expected
12+
// namespace and one of the expected names.
13+
type SecretNamePredicate struct {
14+
predicate.Funcs
15+
Namespace string
16+
SecretNames []string
17+
}
18+
19+
// Create filters CreateEvents based on the Secret name.
20+
func (sp SecretNamePredicate) Create(e event.CreateEvent) bool {
21+
if e.Object == nil {
22+
return false
23+
}
24+
25+
if secret, ok := e.Object.(*corev1.Secret); ok {
26+
return secretMatches(secret, sp.Namespace, sp.SecretNames)
27+
}
28+
29+
return false
30+
}
31+
32+
// Update filters UpdateEvents based on the Secret name.
33+
func (sp SecretNamePredicate) Update(e event.UpdateEvent) bool {
34+
if e.ObjectNew == nil {
35+
return false
36+
}
37+
38+
if secret, ok := e.ObjectNew.(*corev1.Secret); ok {
39+
return secretMatches(secret, sp.Namespace, sp.SecretNames)
40+
}
41+
42+
return false
43+
}
44+
45+
// Delete filters DeleteEvents based on the Secret name.
46+
func (sp SecretNamePredicate) Delete(e event.DeleteEvent) bool {
47+
if e.Object == nil {
48+
return false
49+
}
50+
51+
if secret, ok := e.Object.(*corev1.Secret); ok {
52+
return secretMatches(secret, sp.Namespace, sp.SecretNames)
53+
}
54+
55+
return false
56+
}
57+
58+
// Generic filters GenericEvents based on the Secret name.
59+
func (sp SecretNamePredicate) Generic(e event.GenericEvent) bool {
60+
if e.Object == nil {
61+
return false
62+
}
63+
64+
if secret, ok := e.Object.(*corev1.Secret); ok {
65+
return secretMatches(secret, sp.Namespace, sp.SecretNames)
66+
}
67+
68+
return false
69+
}
70+
71+
func secretMatches(secret *corev1.Secret, namespace string, names []string) bool {
72+
if secret.GetNamespace() != namespace {
73+
return false
74+
}
75+
76+
return slices.Contains(names, secret.GetName())
77+
}

‎internal/framework/controller/predicate/secret_test.go

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package predicate
2+
3+
import (
4+
"testing"
5+
6+
. "github.com/onsi/gomega"
7+
corev1 "k8s.io/api/core/v1"
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"sigs.k8s.io/controller-runtime/pkg/event"
10+
)
11+
12+
func TestSecretNamePredicate(t *testing.T) {
13+
t.Parallel()
14+
15+
pred := SecretNamePredicate{
16+
Namespace: "test-namespace",
17+
SecretNames: []string{"secret1", "secret2"},
18+
}
19+
20+
tests := []struct {
21+
createEvent *event.CreateEvent
22+
updateEvent *event.UpdateEvent
23+
deleteEvent *event.DeleteEvent
24+
genericEvent *event.GenericEvent
25+
name string
26+
expUpdate bool
27+
}{
28+
{
29+
name: "Create event with matching secret",
30+
createEvent: &event.CreateEvent{
31+
Object: &corev1.Secret{
32+
ObjectMeta: metav1.ObjectMeta{
33+
Name: "secret1",
34+
Namespace: "test-namespace",
35+
},
36+
},
37+
},
38+
expUpdate: true,
39+
},
40+
{
41+
name: "Create event with non-matching secret",
42+
createEvent: &event.CreateEvent{
43+
Object: &corev1.Secret{
44+
ObjectMeta: metav1.ObjectMeta{
45+
Name: "secret3",
46+
Namespace: "test-namespace",
47+
},
48+
},
49+
},
50+
expUpdate: false,
51+
},
52+
{
53+
name: "Create event with non-matching namespace",
54+
createEvent: &event.CreateEvent{
55+
Object: &corev1.Secret{
56+
ObjectMeta: metav1.ObjectMeta{
57+
Name: "secret1",
58+
Namespace: "other-namespace",
59+
},
60+
},
61+
},
62+
expUpdate: false,
63+
},
64+
{
65+
name: "Update event with matching secret",
66+
updateEvent: &event.UpdateEvent{
67+
ObjectNew: &corev1.Secret{
68+
ObjectMeta: metav1.ObjectMeta{
69+
Name: "secret2",
70+
Namespace: "test-namespace",
71+
},
72+
},
73+
},
74+
expUpdate: true,
75+
},
76+
{
77+
name: "Update event with non-matching secret",
78+
updateEvent: &event.UpdateEvent{
79+
ObjectNew: &corev1.Secret{
80+
ObjectMeta: metav1.ObjectMeta{
81+
Name: "secret3",
82+
Namespace: "test-namespace",
83+
},
84+
},
85+
},
86+
expUpdate: false,
87+
},
88+
{
89+
name: "Update event with non-matching namespace",
90+
updateEvent: &event.UpdateEvent{
91+
ObjectNew: &corev1.Secret{
92+
ObjectMeta: metav1.ObjectMeta{
93+
Name: "secret1",
94+
Namespace: "other-namespace",
95+
},
96+
},
97+
},
98+
expUpdate: false,
99+
},
100+
{
101+
name: "Delete event with matching secret",
102+
deleteEvent: &event.DeleteEvent{
103+
Object: &corev1.Secret{
104+
ObjectMeta: metav1.ObjectMeta{
105+
Name: "secret1",
106+
Namespace: "test-namespace",
107+
},
108+
},
109+
},
110+
expUpdate: true,
111+
},
112+
{
113+
name: "Delete event with non-matching secret",
114+
deleteEvent: &event.DeleteEvent{
115+
Object: &corev1.Secret{
116+
ObjectMeta: metav1.ObjectMeta{
117+
Name: "secret3",
118+
Namespace: "test-namespace",
119+
},
120+
},
121+
},
122+
expUpdate: false,
123+
},
124+
{
125+
name: "Delete event with non-matching namespace",
126+
deleteEvent: &event.DeleteEvent{
127+
Object: &corev1.Secret{
128+
ObjectMeta: metav1.ObjectMeta{
129+
Name: "secret1",
130+
Namespace: "other-namespace",
131+
},
132+
},
133+
},
134+
expUpdate: false,
135+
},
136+
{
137+
name: "Generic event with matching secret",
138+
genericEvent: &event.GenericEvent{
139+
Object: &corev1.Secret{
140+
ObjectMeta: metav1.ObjectMeta{
141+
Name: "secret1",
142+
Namespace: "test-namespace",
143+
},
144+
},
145+
},
146+
expUpdate: true,
147+
},
148+
{
149+
name: "Generic event with non-matching secret",
150+
genericEvent: &event.GenericEvent{
151+
Object: &corev1.Secret{
152+
ObjectMeta: metav1.ObjectMeta{
153+
Name: "secret3",
154+
Namespace: "test-namespace",
155+
},
156+
},
157+
},
158+
expUpdate: false,
159+
},
160+
{
161+
name: "Generic event with non-matching namespace",
162+
genericEvent: &event.GenericEvent{
163+
Object: &corev1.Secret{
164+
ObjectMeta: metav1.ObjectMeta{
165+
Name: "secret1",
166+
Namespace: "other-namespace",
167+
},
168+
},
169+
},
170+
expUpdate: false,
171+
},
172+
}
173+
174+
for _, test := range tests {
175+
t.Run(test.name, func(t *testing.T) {
176+
t.Parallel()
177+
g := NewWithT(t)
178+
179+
var result bool
180+
switch {
181+
case test.createEvent != nil:
182+
result = pred.Create(*test.createEvent)
183+
case test.updateEvent != nil:
184+
result = pred.Update(*test.updateEvent)
185+
case test.deleteEvent != nil:
186+
result = pred.Delete(*test.deleteEvent)
187+
default:
188+
result = pred.Generic(*test.genericEvent)
189+
}
190+
191+
g.Expect(test.expUpdate).To(Equal(result))
192+
})
193+
}
194+
}

‎internal/framework/controller/resource.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
11
package controller
22

3-
import "fmt"
3+
import (
4+
"fmt"
5+
6+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7+
"k8s.io/apimachinery/pkg/types"
8+
)
49

510
// CreateNginxResourceName creates the base resource name for all nginx resources
611
// created by the control plane.
712
func CreateNginxResourceName(prefix, suffix string) string {
813
return fmt.Sprintf("%s-%s", prefix, suffix)
914
}
15+
16+
// ObjectMetaToNamespacedName converts ObjectMeta to NamespacedName.
17+
func ObjectMetaToNamespacedName(meta metav1.ObjectMeta) types.NamespacedName {
18+
return types.NamespacedName{
19+
Namespace: meta.Namespace,
20+
Name: meta.Name,
21+
}
22+
}

‎internal/mode/static/provisioner/eventloop.go

+26-3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/nginx/nginx-gateway-fabric/internal/framework/controller/predicate"
1919
"github.com/nginx/nginx-gateway-fabric/internal/framework/events"
2020
ngftypes "github.com/nginx/nginx-gateway-fabric/internal/framework/types"
21+
"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
2122
)
2223

2324
func newEventLoop(
@@ -26,11 +27,30 @@ func newEventLoop(
2627
handler *eventHandler,
2728
logger logr.Logger,
2829
selector metav1.LabelSelector,
30+
ngfNamespace string,
31+
dockerSecrets []string,
32+
usageConfig *config.UsageReportConfig,
2933
) (*events.EventLoop, error) {
3034
nginxResourceLabelPredicate := predicate.NginxLabelPredicate(selector)
3135

36+
secretsToWatch := make([]string, 0, len(dockerSecrets)+3)
37+
secretsToWatch = append(secretsToWatch, dockerSecrets...)
38+
39+
if usageConfig != nil {
40+
if usageConfig.SecretName != "" {
41+
secretsToWatch = append(secretsToWatch, usageConfig.SecretName)
42+
}
43+
if usageConfig.CASecretName != "" {
44+
secretsToWatch = append(secretsToWatch, usageConfig.CASecretName)
45+
}
46+
if usageConfig.ClientSSLSecretName != "" {
47+
secretsToWatch = append(secretsToWatch, usageConfig.ClientSSLSecretName)
48+
}
49+
}
50+
3251
controllerRegCfgs := []struct {
3352
objectType ngftypes.ObjectType
53+
name string
3454
options []controller.Option
3555
}{
3656
{
@@ -85,15 +105,18 @@ func newEventLoop(
85105
options: []controller.Option{
86106
controller.WithK8sPredicate(
87107
k8spredicate.And(
88-
k8spredicate.GenerationChangedPredicate{},
89-
nginxResourceLabelPredicate,
108+
k8spredicate.ResourceVersionChangedPredicate{},
109+
k8spredicate.Or(
110+
nginxResourceLabelPredicate,
111+
predicate.SecretNamePredicate{Namespace: ngfNamespace, SecretNames: secretsToWatch},
112+
),
90113
),
91114
),
92115
},
93116
},
94117
}
95118

96-
eventCh := make(chan interface{})
119+
eventCh := make(chan any)
97120
for _, regCfg := range controllerRegCfgs {
98121
gvk, err := apiutil.GVKForObject(regCfg.objectType, mgr.GetScheme())
99122
if err != nil {

0 commit comments

Comments
 (0)