Surface env history and on-demand stale Secret diagnostics

internal/issues/category.go

@@ -149,7 +149,7 @@ func classifyProblem(in classifyInput) issuesapi.Category {
 	switch in.Reason {
 	case "CoreDNS NXDOMAIN override", "CoreDNS service DNS rewrite":
 		return issuesapi.CategoryDNSFailure
-	case "DuplicateEnvVar":
+	case "DuplicateEnvVar", "StaleSecretEnv":
 		return issuesapi.CategoryInvalidConfiguration
 	case "Missing referenced Service":
 		return issuesapi.CategoryMissingConfigRef

internal/issues/category_test.go

@@ -33,6 +33,7 @@ func TestClassify(t *testing.T) {
 		{"deployment duplicate env", classifyInput{Source: SourceProblem, Kind: "Deployment", Reason: "DuplicateEnvVar"}, issuesapi.CategoryInvalidConfiguration},
 		{"job duplicate env is not job failure", classifyInput{Source: SourceProblem, Kind: "Job", Reason: "DuplicateEnvVar"}, issuesapi.CategoryInvalidConfiguration},
 		{"cronjob duplicate env", classifyInput{Source: SourceProblem, Kind: "CronJob", Reason: "DuplicateEnvVar"}, issuesapi.CategoryInvalidConfiguration},
+		{"stale Secret env", classifyInput{Source: SourceProblem, Kind: "Pod", Reason: "StaleSecretEnv"}, issuesapi.CategoryInvalidConfiguration},
 
 		// problem / Pod
 		{"image pull backoff", classifyInput{Source: SourceProblem, Kind: "Pod", Reason: "ImagePullBackOff"}, issuesapi.CategoryImagePullFailed},

internal/issues/issues_test.go

@@ -222,6 +222,28 @@ func TestCompose_GroupsMemberPodsUnderOwner(t *testing.T) {
 	}
 }
 
+func TestCompose_StaleSecretEnvSurvivesAsStableWorkloadIssue(t *testing.T) {
+	p := &fakeProvider{problems: []k8s.Detection{{
+		Kind: "Pod", Namespace: "shop", Name: "catalog-abc-1", Severity: "warning",
+		Reason: "StaleSecretEnv", Message: "container may hold a stale Secret-backed env value",
+		OwnerGroup: "apps", OwnerKind: "Deployment", OwnerName: "catalog",
+		Fingerprint: "stale-secret-env",
+	}}}
+	filters := Filters{Grouped: true}
+	first := Compose(p, filters)
+	second := Compose(p, filters)
+	if len(first) != 1 || len(second) != 1 {
+		t.Fatalf("stale Secret env issue was dropped during compose: first=%+v second=%+v", first, second)
+	}
+	issue := first[0]
+	if issue.Category != issuesapi.CategoryInvalidConfiguration || issue.Group != "apps" || issue.Kind != "Deployment" || issue.Namespace != "shop" || issue.Name != "catalog" {
+		t.Fatalf("unexpected grouped stale Secret env issue: %+v", issue)
+	}
+	if issue.ID == "" || issue.ID != second[0].ID || issue.Fingerprint != "stale-secret-env" {
+		t.Fatalf("stale Secret env issue identity is not stable: first=%+v second=%+v", first, second)
+	}
+}
+
 func TestCompose_GroupedKindFilterMatchesSubject(t *testing.T) {
 	// A crashlooping Deployment is evidenced by Pod rows that fold under the
 	// Deployment subject. On the GROUPED surface, kind=Deployment must return

internal/k8s/cache.go

@@ -1,7 +1,9 @@
 package k8s
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
@@ -116,6 +118,185 @@ type ResourceCache struct {
 	// continuously OutOfSync. Lives here so its lifecycle is the cache's:
 	// recreated per cluster, dropped on a kubeconfig context switch.
 	argoDrift *argoDriftTracker
+	// secretWriteTimes preserves verified Secret data-owner write times that
+	// the shared informer transform intentionally strips before caching.
+	secretWriteTimes *secretDataManagerWriteIndex
+}
+
+type secretDataManagerWrite struct {
+	uid string
+	at  time.Time
+}
+
+type secretDataManagerWriteVersion struct {
+	namespace       string
+	name            string
+	uid             string
+	resourceVersion string
+}
+
+type pendingSecretDataManagerWrite struct {
+	write *secretDataManagerWrite
+	helm  bool
+}
+
+type secretDataManagerWriteIndex struct {
+	mu           sync.Mutex
+	committed    map[string]secretDataManagerWrite
+	pending      map[secretDataManagerWriteVersion]pendingSecretDataManagerWrite
+	pendingLimit int
+}
+
+const maxPendingSecretDataManagerWrites = 10_000
+
+func newSecretDataManagerWriteIndex() *secretDataManagerWriteIndex {
+	return &secretDataManagerWriteIndex{
+		committed:    make(map[string]secretDataManagerWrite),
+		pending:      make(map[secretDataManagerWriteVersion]pendingSecretDataManagerWrite),
+		pendingLimit: maxPendingSecretDataManagerWrites,
+	}
+}
+
+func secretWriteKey(namespace, name string) string {
+	return namespace + "\x00" + name
+}
+
+func secretChangeWritesData(change k8score.ResourceChange) bool {
+	if change.Diff == nil {
+		return false
+	}
+	for _, field := range change.Diff.Fields {
+		if isSecretDataWritePath(field.Path) {
+			return true
+		}
+	}
+	return false
+}
+
+func isSecretDataWritePath(path string) bool {
+	switch path {
+	case "data (added keys)", "data (removed keys)", "data (modified keys)",
+		"stringData (added keys)", "stringData (removed keys)", "stringData (modified keys)":
+		return true
+	default:
+		return false
+	}
+}
+
+func (index *secretDataManagerWriteIndex) capture(obj any) {
+	secret, ok := obj.(*corev1.Secret)
+	if !ok || secret == nil || index == nil {
+		return
+	}
+	var latest time.Time
+	for _, entry := range secret.ManagedFields {
+		if entry.Time == nil || entry.FieldsV1 == nil ||
+			(entry.Operation != metav1.ManagedFieldsOperationApply && entry.Operation != metav1.ManagedFieldsOperationUpdate) {
+			continue
+		}
+		raw := entry.FieldsV1.Raw
+		if !bytes.Contains(raw, []byte(`"f:data"`)) && !bytes.Contains(raw, []byte(`"f:stringData"`)) {
+			continue
+		}
+		var fields map[string]json.RawMessage
+		if json.Unmarshal(raw, &fields) != nil {
+			continue
+		}
+		if _, ownsData := fields["f:data"]; !ownsData {
+			if _, ownsStringData := fields["f:stringData"]; !ownsStringData {
+				continue
+			}
+		}
+		if entry.Time.Time.After(latest) {
+			latest = entry.Time.Time
+		}
+	}
+	version := secretDataManagerWriteVersion{
+		namespace:       secret.Namespace,
+		name:            secret.Name,
+		uid:             string(secret.UID),
+		resourceVersion: secret.ResourceVersion,
+	}
+	pending := pendingSecretDataManagerWrite{
+		helm: secret.Type == corev1.SecretType("helm.sh/release.v1"),
+	}
+	if !latest.IsZero() {
+		pending.write = &secretDataManagerWrite{uid: string(secret.UID), at: latest}
+	}
+	index.mu.Lock()
+	if _, exists := index.pending[version]; !exists && len(index.pending) >= index.pendingLimit {
+		// Transform observations are advisory. Under callback starvation,
+		// discard unmatched candidates rather than grow without bound; a
+		// missing fallback fails closed and a later object version can restage.
+		clear(index.pending)
+	}
+	index.pending[version] = pending
+	index.mu.Unlock()
+}
+
+func (index *secretDataManagerWriteIndex) reconcile(change k8score.ResourceChange, obj any) {
+	if index == nil || change.Kind != "Secret" {
+		return
+	}
+	key := secretWriteKey(change.Namespace, change.Name)
+	index.mu.Lock()
+	defer index.mu.Unlock()
+
+	if change.Operation == k8score.OpDelete {
+		if write, ok := index.committed[key]; ok && write.uid == change.UID {
+			delete(index.committed, key)
+		}
+		for version := range index.pending {
+			if version.namespace == change.Namespace && version.name == change.Name && version.uid == change.UID {
+				delete(index.pending, version)
+			}
+		}
+		return
+	}
+
+	meta, ok := obj.(metav1.Object)
+	if !ok || meta == nil {
+		return
+	}
+	version := secretDataManagerWriteVersion{
+		namespace:       change.Namespace,
+		name:            change.Name,
+		uid:             change.UID,
+		resourceVersion: meta.GetResourceVersion(),
+	}
+	pending, ok := index.pending[version]
+	if !ok {
+		return
+	}
+	delete(index.pending, version)
+
+	if pending.helm {
+		if write, ok := index.committed[key]; !ok || write.uid == change.UID {
+			delete(index.committed, key)
+		}
+		return
+	}
+	if change.Operation == k8score.OpUpdate && !secretChangeWritesData(change) {
+		return
+	}
+	if write, ok := index.committed[key]; ok && write.uid != change.UID && change.Operation != k8score.OpAdd {
+		return
+	}
+	if pending.write == nil {
+		delete(index.committed, key)
+		return
+	}
+	index.committed[key] = *pending.write
+}
+
+func (index *secretDataManagerWriteIndex) load(namespace, name string) (secretDataManagerWrite, bool) {
+	if index == nil {
+		return secretDataManagerWrite{}, false
+	}
+	index.mu.Lock()
+	defer index.mu.Unlock()
+	write, ok := index.committed[secretWriteKey(namespace, name)]
+	return write, ok
 }
 
 var (
@@ -174,6 +355,7 @@ func InitResourceCache(ctx context.Context) error {
 		// wiring-time value keeps late events truthfully attributed to the
 		// cluster they came from.
 		recordClusterContext := ActiveClusterContext()
+		secretWriteTimes := newSecretDataManagerWriteIndex()
 
 		cfg := k8score.CacheConfig{
 			Client:                  k8sClient,
@@ -193,7 +375,12 @@ func InitResourceCache(ctx context.Context) error {
 				timeline.IncrementReceived(kind)
 			},
 
+			OnTransform: func(obj any) {
+				secretWriteTimes.capture(obj)
+			},
+
 			OnChange: func(change k8score.ResourceChange, obj, oldObj any) {
+				secretWriteTimes.reconcile(change, obj)
 				if DebugEvents && change.Operation == "add" &&
 					(change.Kind == "Pod" || change.Kind == "Deployment" || change.Kind == "Service") {
 					log.Printf("[DEBUG] enqueueChange: %s add %s/%s", change.Kind, change.Namespace, change.Name)
@@ -235,9 +422,10 @@ func InitResourceCache(ctx context.Context) error {
 		initialSyncComplete = core.IsSyncComplete()
 
 		resourceCache = &ResourceCache{
-			ResourceCache:  core,
-			secretsEnabled: scopes["secrets"].Enabled,
-			argoDrift:      newArgoDriftTracker(),
+			ResourceCache:    core,
+			secretsEnabled:   scopes["secrets"].Enabled,
+			argoDrift:        newArgoDriftTracker(),
+			secretWriteTimes: secretWriteTimes,
 		}
 	})
 	return initErr

internal/k8s/detect_config.go

@@ -32,6 +32,7 @@ func detectConfigProblems(cache *ResourceCache, namespace string, now time.Time)
 	}
 	out = append(out, detectEnvServiceRefs(cache, namespace, now)...)
 	out = append(out, detectDuplicateEnvVars(cache, namespace, now)...)
+	out = append(out, detectStaleSecretEnv(cache, namespace, now)...)
 	return out
 }