Skip to content

Commit a085db6

Browse files
committed
feat(k8score): idle reaper for namespace-scoped dynamic informers
Per-namespace dynamic informers (the B2.1 model) can accumulate over the life of a long-running, namespace-restricted deployment as different namespaces get watched. Add an idle reaper that stops a namespace-scoped informer after InformerIdleTTL (default 30m) without a read, re-creating it transparently on the next access. Liveness is the read path itself: List/Get/Count/EnsureWatching stamp the entry's lastAccess. The frontend re-lists open views every 60-120s (React Query refetchInterval), and SSE topology rebuilds also list — both flow through that stamp — so an open view's informer is never reaped out from under it, and an explicit SSE lease/refcount is unnecessary. Cluster-wide informers (one per GVR, serve every namespace) are exempt; a negative TTL disables reaping. To make eviction safe, informers are now constructed directly via NewFilteredDynamicInformer instead of through a shared factory: a factory caches one informer per GVR and would hand back the stopped instance after a reap (a SharedInformer cannot be re-Run). Standalone informers, each under its own cancelable context, can be stopped and re-created freely. This also removes the per-namespace factory map and simplifies Stop(). Builds on #770.
1 parent ee22b11 commit a085db6

3 files changed

Lines changed: 240 additions & 56 deletions

File tree

pkg/k8score/cache_test.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1347,6 +1347,108 @@ func TestDynamicResourceCache_StatusAPIsSpanNamespaceInformers(t *testing.T) {
13471347
}
13481348
}
13491349

1350+
func (d *DynamicResourceCache) hasInformer(gvr schema.GroupVersionResource, ns string) bool {
1351+
d.mu.RLock()
1352+
defer d.mu.RUnlock()
1353+
_, ok := d.informers[informerKey{gvr: gvr, ns: ns}]
1354+
return ok
1355+
}
1356+
1357+
// The idle reaper stops namespace-scoped informers untouched beyond the TTL,
1358+
// leaves freshly-read ones alone, and never touches cluster-wide informers.
1359+
// Reaped informers are re-created transparently on the next access.
1360+
func TestDynamicResourceCache_ReapsIdleNamespacedInformers(t *testing.T) {
1361+
const nsIdle, nsFresh = "idle-ns", "fresh-ns"
1362+
gvr := schema.GroupVersionResource{Group: "example.com", Version: "v1", Resource: "widgets"}
1363+
dyn := fakeDynamicForListAccess(t, map[schema.GroupVersionResource]string{
1364+
gvr: "WidgetList",
1365+
}, func(_ schema.GroupVersionResource, ns string) bool { return ns != "" }) // namespaced only
1366+
1367+
d, err := NewDynamicResourceCache(DynamicCacheConfig{DynamicClient: dyn, InformerIdleTTL: time.Hour})
1368+
if err != nil {
1369+
t.Fatalf("NewDynamicResourceCache failed: %v", err)
1370+
}
1371+
defer d.Stop()
1372+
1373+
for _, ns := range []string{nsIdle, nsFresh} {
1374+
if _, err := d.ListBlocking(gvr, ns, 2*time.Second); err != nil {
1375+
t.Fatalf("ListBlocking(%q) failed: %v", ns, err)
1376+
}
1377+
}
1378+
1379+
// Backdate the idle informer's last access; leave the fresh one current.
1380+
d.mu.RLock()
1381+
d.informers[informerKey{gvr: gvr, ns: nsIdle}].lastAccess.Store(time.Now().Add(-2 * time.Hour).UnixNano())
1382+
d.mu.RUnlock()
1383+
1384+
if reaped := d.reapIdleInformers(time.Now()); reaped != 1 {
1385+
t.Fatalf("reaped %d informers, want 1", reaped)
1386+
}
1387+
if d.hasInformer(gvr, nsIdle) {
1388+
t.Errorf("idle informer for %q was not reaped", nsIdle)
1389+
}
1390+
if !d.hasInformer(gvr, nsFresh) {
1391+
t.Errorf("fresh informer for %q was wrongly reaped", nsFresh)
1392+
}
1393+
1394+
// Next access re-creates the reaped informer.
1395+
if _, err := d.ListBlocking(gvr, nsIdle, 2*time.Second); err != nil {
1396+
t.Fatalf("re-list after reap failed: %v", err)
1397+
}
1398+
if !d.hasInformer(gvr, nsIdle) {
1399+
t.Errorf("informer for %q was not re-created on access", nsIdle)
1400+
}
1401+
}
1402+
1403+
// Cluster-wide informers are exempt from reaping even when idle, and a
1404+
// negative TTL disables reaping entirely.
1405+
func TestDynamicResourceCache_ReapExemptionsAndDisable(t *testing.T) {
1406+
gvr := schema.GroupVersionResource{Group: "example.com", Version: "v1", Resource: "gadgets"}
1407+
1408+
t.Run("cluster-wide exempt", func(t *testing.T) {
1409+
dyn := fakeDynamicForListAccess(t, map[schema.GroupVersionResource]string{
1410+
gvr: "GadgetList",
1411+
}, func(_ schema.GroupVersionResource, ns string) bool { return ns == "" }) // cluster-wide allowed
1412+
d, err := NewDynamicResourceCache(DynamicCacheConfig{DynamicClient: dyn, InformerIdleTTL: time.Hour})
1413+
if err != nil {
1414+
t.Fatalf("NewDynamicResourceCache failed: %v", err)
1415+
}
1416+
defer d.Stop()
1417+
if _, err := d.ListBlocking(gvr, "", 2*time.Second); err != nil {
1418+
t.Fatalf("ListBlocking failed: %v", err)
1419+
}
1420+
d.mu.RLock()
1421+
d.informers[informerKey{gvr: gvr}].lastAccess.Store(int64(0)) // ancient
1422+
d.mu.RUnlock()
1423+
if reaped := d.reapIdleInformers(time.Now()); reaped != 0 {
1424+
t.Errorf("reaped %d cluster-wide informers, want 0 (exempt)", reaped)
1425+
}
1426+
if !d.hasInformer(gvr, "") {
1427+
t.Error("cluster-wide informer was reaped")
1428+
}
1429+
})
1430+
1431+
t.Run("negative TTL disables reaping", func(t *testing.T) {
1432+
dyn := fakeDynamicForListAccess(t, map[schema.GroupVersionResource]string{
1433+
gvr: "GadgetList",
1434+
}, func(_ schema.GroupVersionResource, ns string) bool { return ns != "" }) // namespaced only
1435+
d, err := NewDynamicResourceCache(DynamicCacheConfig{DynamicClient: dyn, InformerIdleTTL: -1})
1436+
if err != nil {
1437+
t.Fatalf("NewDynamicResourceCache failed: %v", err)
1438+
}
1439+
defer d.Stop()
1440+
if _, err := d.ListBlocking(gvr, "team-x", 2*time.Second); err != nil {
1441+
t.Fatalf("ListBlocking failed: %v", err)
1442+
}
1443+
d.mu.RLock()
1444+
d.informers[informerKey{gvr: gvr, ns: "team-x"}].lastAccess.Store(int64(0))
1445+
d.mu.RUnlock()
1446+
if reaped := d.reapIdleInformers(time.Now()); reaped != 0 {
1447+
t.Errorf("reaped %d with reaping disabled, want 0", reaped)
1448+
}
1449+
})
1450+
}
1451+
13501452
// EnsureWatching must surface a probe that's forbidden everywhere it looks
13511453
// (cluster-wide AND the fallback namespace) as an apierrors.IsForbidden-
13521454
// classifiable error through the %w wrap — that's what lets the resources

0 commit comments

Comments
 (0)