Skip to content

Commit 52c74f9

Browse files
authored
Fix generation increments on operator tick (#86)
Only update deployments / statefulsets if something meaningful has changed - IE the resource spec or its configuration. Each tick of the operator is causing higher order kube resources - deployments and stateful sets - to increment their generation. The operator unconditionally applies an update against the resource on each tick - even when nothing about the actual and desired state has changed. This is mostly fine, but it occasionally - when using something like prometheus - causes monitors fail indicating that the deployment or stateful set has failed a rollout because the generations don't match -- IE a race condition between scrape time and the update.
1 parent a655638 commit 52c74f9

File tree

4 files changed

+393
-2
lines changed

4 files changed

+393
-2
lines changed

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ Also check this project's [releases](https://github.com/powerhome/redis-operator
99

1010
## Unreleased
1111

12+
### Fixed
13+
- [Fix unnecessary kube deployment and statefulset generation increment on operator tick](https://github.com/powerhome/redis-operator/pull/86)
14+
1215
## [v4.4.0] - 2025-12-19
1316

1417
### Changed

operator/redisfailover/service/client.go

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package service
22

33
import (
4+
"crypto/sha256"
5+
"encoding/hex"
6+
"encoding/json"
47
"fmt"
8+
9+
appsv1 "k8s.io/api/apps/v1"
510
"k8s.io/apimachinery/pkg/api/errors"
611
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
712
"k8s.io/apimachinery/pkg/util/intstr"
@@ -143,12 +148,59 @@ func (r *RedisFailoverKubeClient) EnsureHAProxyRedisMasterDeployment(rf *redisfa
143148
}
144149
d.Spec.Template.Annotations[haproxyConfigChecksumAnnotationKey] = digest
145150

151+
// Compute a digest of the desired spec and store it on the
152+
// deployment's metadata annotations. On each reconcile we
153+
// re-generate the desired spec, re-hash it, and compare
154+
// against the stored annotation — comparison is O(1). This
155+
// also automatically catches any future additions to
156+
// generateHAProxyRedisMasterDeployment without needing manual
157+
// updates here.
158+
specDigest, specErr := specDigest(d.Spec)
159+
if specErr != nil {
160+
return fmt.Errorf("EnsureHAProxyRedisMasterDeployment failed to compute spec digest: %w", specErr)
161+
}
162+
if existing, getErr := r.K8SService.GetDeployment(rf.Namespace, d.Name); getErr == nil {
163+
if existing.Annotations[haproxyDeploymentSpecChecksumKey] == specDigest {
164+
return nil
165+
}
166+
}
167+
if d.Annotations == nil {
168+
d.Annotations = make(map[string]string)
169+
}
170+
d.Annotations[haproxyDeploymentSpecChecksumKey] = specDigest
171+
146172
err = r.K8SService.CreateOrUpdateDeployment(rf.Namespace, d)
147173

148174
r.setEnsureOperationMetrics(d.Namespace, d.Name, "EnsureHAProxyRedisMasterDeployment", rf.Name, err)
149175
return err
150176
}
151177

178+
// specDigest returns a stable SHA-256 hex digest of any resource Spec
179+
// value. encoding/json marshals struct fields in declaration order
180+
// and map keys in sorted order (since Go 1.12), so the output is
181+
// deterministic for a given input. managedSpec is a type constraint
182+
// that enumerates the Kubernetes resource spec structs whose digests
183+
// the operator tracks. Listing types expelicitly here means:
184+
//
185+
// - the compiler rejects any accidental wrong-type call site - nil
186+
// can never be passed (struct types are not nilable)
187+
//
188+
// - adding a new Ensure* method for a new resource type requires
189+
// updating this list, which serves as a forcing function to
190+
// remember to add the skip-update guard too
191+
type managedSpec interface {
192+
appsv1.DeploymentSpec | appsv1.StatefulSetSpec
193+
}
194+
195+
func specDigest[T managedSpec](spec T) (string, error) {
196+
data, err := json.Marshal(spec)
197+
if err != nil {
198+
return "", err
199+
}
200+
sum := sha256.Sum256(data)
201+
return hex.EncodeToString(sum[:]), nil
202+
}
203+
152204
// EnsureSentinelService makes sure the sentinel service exists
153205
func (r *RedisFailoverKubeClient) EnsureSentinelService(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) error {
154206
svc := generateSentinelService(rf, labels, ownerRefs)
@@ -173,8 +225,22 @@ func (r *RedisFailoverKubeClient) EnsureSentinelDeployment(rf *redisfailoverv1.R
173225
}
174226
}
175227
d := generateSentinelDeployment(rf, labels, ownerRefs)
176-
err := r.K8SService.CreateOrUpdateDeployment(rf.Namespace, d)
177228

229+
digest, err := specDigest(d.Spec)
230+
if err != nil {
231+
return fmt.Errorf("EnsureSentinelDeployment failed to compute spec digest: %w", err)
232+
}
233+
if existing, getErr := r.K8SService.GetDeployment(rf.Namespace, d.Name); getErr == nil {
234+
if existing.Annotations[sentinelDeploymentSpecChecksumKey] == digest {
235+
return nil
236+
}
237+
}
238+
if d.Annotations == nil {
239+
d.Annotations = make(map[string]string)
240+
}
241+
d.Annotations[sentinelDeploymentSpecChecksumKey] = digest
242+
243+
err = r.K8SService.CreateOrUpdateDeployment(rf.Namespace, d)
178244
r.setEnsureOperationMetrics(d.Namespace, d.Name, "Deployment", rf.Name, err)
179245
return err
180246
}
@@ -300,8 +366,22 @@ func (r *RedisFailoverKubeClient) EnsureRedisStatefulset(rf *redisfailoverv1.Red
300366
}
301367
}
302368
ss := generateRedisStatefulSet(rf, labels, ownerRefs)
303-
err := r.K8SService.CreateOrUpdateStatefulSet(rf.Namespace, ss)
304369

370+
digest, err := specDigest(ss.Spec)
371+
if err != nil {
372+
return fmt.Errorf("EnsureRedisStatefulset failed to compute spec digest: %w", err)
373+
}
374+
if existing, getErr := r.K8SService.GetStatefulSet(rf.Namespace, ss.Name); getErr == nil {
375+
if existing.Annotations[redisStatefulSetSpecChecksumKey] == digest {
376+
return nil
377+
}
378+
}
379+
if ss.Annotations == nil {
380+
ss.Annotations = make(map[string]string)
381+
}
382+
ss.Annotations[redisStatefulSetSpecChecksumKey] = digest
383+
384+
err = r.K8SService.CreateOrUpdateStatefulSet(rf.Namespace, ss)
305385
r.setEnsureOperationMetrics(ss.Namespace, ss.Name, "StatefulSet", rf.Name, err)
306386
return err
307387
}

operator/redisfailover/service/constants.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,7 @@ const (
4141

4242
const (
4343
haproxyConfigChecksumAnnotationKey = "checksum/haproxy-cfg"
44+
haproxyDeploymentSpecChecksumKey = "checksum/haproxy-deployment-spec"
45+
sentinelDeploymentSpecChecksumKey = "checksum/sentinel-deployment-spec"
46+
redisStatefulSetSpecChecksumKey = "checksum/redis-statefulset-spec"
4447
)

0 commit comments

Comments
 (0)