Skip to content

Commit e5abb96

Browse files
kyledong-susekyledong-suse
authored
Merge pull request #757 from kyledong-suse/feature
feat(metrics): export runtime_enforcer_active_violations gauge
2 parents 18246f8 + 4a5e31a commit e5abb96

14 files changed

Lines changed: 474 additions & 99 deletions

File tree

‎charts/runtime-enforcer/templates/otel-collector/configmap.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ data:
4949
receivers: [otlp]
5050
exporters: [count, debug]
5151
metrics:
52-
receivers: [count]
52+
receivers: [count, otlp]
5353
processors: [deltatocumulative]
5454
exporters: [prometheus]
5555
{{- end }}

‎cmd/controller/main.go‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"context"
55
"crypto/tls"
6+
"errors"
67
"flag"
78
"fmt"
89
"log/slog"
@@ -33,8 +34,10 @@ import (
3334
"github.com/rancher-sandbox/runtime-enforcer/internal/customloggers/httpserverlogger"
3435
"github.com/rancher-sandbox/runtime-enforcer/internal/events"
3536
"github.com/rancher-sandbox/runtime-enforcer/internal/grpcexporter"
37+
"github.com/rancher-sandbox/runtime-enforcer/internal/metrics"
3638

3739
otellog "go.opentelemetry.io/otel/log"
40+
"go.opentelemetry.io/otel/metric"
3841
// +kubebuilder:scaffold:imports
3942
)
4043

@@ -295,26 +298,65 @@ func getMetricsServerOptions(logger *slog.Logger, config *Config) (*certwatcher.
295298
return metricsCertWatcher, metricsServerOptions
296299
}
297300

298-
func setupOtel(ctx context.Context, slogger *slog.Logger, config *Config) (func(context.Context) error, error) {
301+
func setupOtel(
302+
ctx context.Context,
303+
slogger *slog.Logger,
304+
config *Config,
305+
) (func(context.Context) error, error) {
306+
if config.otlpEndpoint == "" {
307+
return func(context.Context) error { return nil }, nil
308+
}
309+
299310
var err error
300-
var eventShutdown func(context.Context) error
301-
if config.otlpEndpoint != "" {
302-
var eventLogger otellog.Logger
303-
eventLogger, eventShutdown, err = events.Init(
304-
ctx,
305-
config.otlpEndpoint,
306-
config.otlpCACert,
307-
config.otlpClientCert,
308-
config.otlpClientKey,
309-
config.otlpProtocol,
310-
)
311-
if err != nil {
312-
return nil, fmt.Errorf("failed to initialize otel: %w", err)
313-
}
314-
config.wpStatusSyncConfig.EventLogger = eventLogger
315-
slogger.InfoContext(ctx, "OTLP telemetry enabled", "endpoint", config.otlpEndpoint)
311+
var eventShutdown, metricShutdown func(context.Context) error
312+
var eventLogger otellog.Logger
313+
var activeViolationsGauge metric.Int64Gauge
314+
var shutdowns []func(context.Context) error
315+
316+
eventLogger, eventShutdown, err = events.Init(
317+
ctx,
318+
config.otlpEndpoint,
319+
config.otlpCACert,
320+
config.otlpClientCert,
321+
config.otlpClientKey,
322+
config.otlpProtocol,
323+
)
324+
if err != nil {
325+
return nil, fmt.Errorf("failed to initialize otel logs: %w", err)
316326
}
317-
return eventShutdown, nil
327+
config.wpStatusSyncConfig.EventLogger = eventLogger
328+
shutdowns = append(shutdowns, eventShutdown)
329+
330+
activeViolationsGauge, metricShutdown, err = metrics.Init(
331+
ctx,
332+
config.otlpEndpoint,
333+
config.otlpCACert,
334+
config.otlpClientCert,
335+
config.otlpClientKey,
336+
config.otlpProtocol,
337+
config.wpStatusSyncConfig.UpdateInterval,
338+
)
339+
if err != nil {
340+
_ = eventShutdown(ctx)
341+
return nil, fmt.Errorf("failed to initialize otel metrics: %w", err)
342+
}
343+
config.wpStatusSyncConfig.ActiveViolationsGauge = activeViolationsGauge
344+
shutdowns = append(shutdowns, metricShutdown)
345+
346+
slogger.InfoContext(ctx, "OTLP telemetry enabled", "endpoint", config.otlpEndpoint)
347+
348+
return func(shutdownCtx context.Context) error {
349+
var errs error
350+
for _, shutdown := range shutdowns {
351+
if shutdown == nil {
352+
continue
353+
}
354+
if shutdownErr := shutdown(shutdownCtx); shutdownErr != nil {
355+
errs = errors.Join(errs, shutdownErr)
356+
}
357+
}
358+
return errs
359+
}, nil
318360
}
319361

320362
func main() {
@@ -366,9 +408,9 @@ func main() {
366408
os.Exit(1)
367409
}
368410

369-
var eventShutdown func(context.Context) error
411+
var otelShutdown func(context.Context) error
370412

371-
eventShutdown, err = setupOtel(ctx, slogger, &config)
413+
otelShutdown, err = setupOtel(ctx, slogger, &config)
372414
if err != nil {
373415
setupLog.Error(err, "failed to setup otel")
374416
os.Exit(1)
@@ -404,9 +446,9 @@ func main() {
404446
os.Exit(1)
405447
}
406448

407-
if eventShutdown != nil {
408-
if err = eventShutdown(ctx); err != nil {
409-
slogger.ErrorContext(ctx, "failed to shutdown violation event pipeline", "error", err)
449+
if otelShutdown != nil {
450+
if err = otelShutdown(ctx); err != nil {
451+
slogger.ErrorContext(ctx, "failed to shutdown otel telemetry pipeline", "error", err)
410452
}
411453
}
412454
}

‎docs/installation/quickstart.adoc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,13 @@ Once violations occur, you will see a counter like:
109109
runtime_enforcer_violations_total{action="monitor",k8s_namespace_name="default",node_name="node-1",policy_name="deploy-opensuse-deployment"} 1
110110
```
111111

112+
Alongside it, the controller exports a gauge of the current number of unresolved
113+
violations per policy, taken from `status.activeViolationCount`:
114+
115+
```
116+
runtime_enforcer_active_violations{k8s_namespace_name="default",policy_name="deploy-opensuse-deployment"} 1
117+
```
118+
112119
=== Summary
113120

114121
At this point, Runtime-Enforcer is up and running. You're now ready to write some policies.

‎go.mod‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,15 @@ require (
1111
github.com/onsi/gomega v1.42.1
1212
github.com/spf13/cobra v1.10.2
1313
github.com/stretchr/testify v1.11.1
14+
go.opentelemetry.io/otel v1.44.0
1415
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0
1516
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0
17+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
18+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0
1619
go.opentelemetry.io/otel/log v0.20.0
20+
go.opentelemetry.io/otel/metric v1.44.0
1721
go.opentelemetry.io/otel/sdk/log v0.20.0
22+
go.opentelemetry.io/otel/sdk/metric v1.44.0
1823
golang.org/x/sync v0.22.0
1924
golang.org/x/sys v0.47.0
2025
golang.org/x/time v0.15.0
@@ -100,10 +105,8 @@ require (
100105
github.com/xlab/treeprint v1.2.0 // indirect
101106
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
102107
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
103-
go.opentelemetry.io/otel v1.44.0 // indirect
104108
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect
105109
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
106-
go.opentelemetry.io/otel/metric v1.44.0 // indirect
107110
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
108111
go.opentelemetry.io/otel/trace v1.44.0 // indirect
109112
go.opentelemetry.io/proto/otlp v1.10.0 // indirect

‎go.sum‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 h1:rydZ9sxbc
229229
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0/go.mod h1:earQ25dooT0Hhspq59DZ8YCC50jWfOlFEeWoxy/P444=
230230
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0 h1:owlhcJ3QO3X0YTDTCcDZ4V+6aVDkWbNmBoQ5NUp7Oww=
231231
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0/go.mod h1:MP4eemTiI9zC8fgg+DYynhYDYf3ba72S376TvP+Ye0Q=
232+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
233+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
234+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA=
235+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo=
232236
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw=
233237
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw=
234238
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto=
@@ -237,6 +241,8 @@ go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJ
237241
go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E=
238242
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
239243
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
244+
go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
245+
go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
240246
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
241247
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
242248
go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY=

‎hack/ci/whitelist.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
"msg": "failed to scrape violations",
1515
"component": "controller"
1616
},
17+
{
18+
"level": "ERROR",
19+
"msg": "failed to export active violations metrics",
20+
"component": "controller"
21+
},
1722
{
1823
"level": "WARN",
1924
"msg": "error during NRI plugin execution, retrying...",

‎internal/controller/workloadpolicystatus_helpers.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
)
1616

1717
// processWorkloadPolicy updates the wp.status and wp.annotation in order to acknowledge a violation.
18+
// On success, *wp is replaced with the patched policy. On failure, *wp is left unchanged.
1819
// NOTE: agent side ignores annotation changes and status change via predicate.GenerationChangedPredicate{}.
1920
func (r *WorkloadPolicyStatusSync) processWorkloadPolicy(
2021
ctx context.Context,
@@ -56,6 +57,7 @@ func (r *WorkloadPolicyStatusSync) processWorkloadPolicy(
5657
if err != nil {
5758
return err
5859
}
60+
*wp = *newPolicy
5961
return nil
6062
}
6163

‎internal/controller/workloadpolicystatus_sync.go‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ import (
1010
"github.com/rancher-sandbox/runtime-enforcer/internal/grpcexporter"
1111
"github.com/rancher-sandbox/runtime-enforcer/internal/types/loglevel"
1212

13+
"go.opentelemetry.io/otel/attribute"
1314
otellog "go.opentelemetry.io/otel/log"
15+
"go.opentelemetry.io/otel/metric"
1416
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1517
"sigs.k8s.io/controller-runtime/pkg/client"
1618
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -24,17 +26,19 @@ import (
2426
type WorkloadPolicyStatusSync struct {
2527
client.Client
2628

27-
agentClientPool *grpcexporter.AgentClientPool
28-
updateInterval time.Duration
29-
logger logr.Logger
30-
eventLogger otellog.Logger
29+
agentClientPool *grpcexporter.AgentClientPool
30+
updateInterval time.Duration
31+
logger logr.Logger
32+
eventLogger otellog.Logger
33+
activeViolationsGauge metric.Int64Gauge
3134
}
3235

3336
// WorkloadPolicyStatusSyncConfig holds the configuration for the WorkloadPolicyStatusSync.
3437
type WorkloadPolicyStatusSyncConfig struct {
35-
AgentPoolConf grpcexporter.AgentClientPoolConfig
36-
UpdateInterval time.Duration
37-
EventLogger otellog.Logger
38+
AgentPoolConf grpcexporter.AgentClientPoolConfig
39+
UpdateInterval time.Duration
40+
EventLogger otellog.Logger
41+
ActiveViolationsGauge metric.Int64Gauge
3842
}
3943

4044
func NewWorkloadPolicyStatusSync(
@@ -51,10 +55,11 @@ func NewWorkloadPolicyStatusSync(
5155
}
5256

5357
return &WorkloadPolicyStatusSync{
54-
Client: c,
55-
agentClientPool: agentClientPool,
56-
updateInterval: config.UpdateInterval,
57-
eventLogger: config.EventLogger,
58+
Client: c,
59+
agentClientPool: agentClientPool,
60+
updateInterval: config.UpdateInterval,
61+
eventLogger: config.EventLogger,
62+
activeViolationsGauge: config.ActiveViolationsGauge,
5863
}, nil
5964
}
6065

@@ -115,6 +120,18 @@ func (r *WorkloadPolicyStatusSync) sync(
115120
"failed to process workload policy",
116121
"policy", policyName,
117122
)
123+
continue
124+
}
125+
126+
if r.activeViolationsGauge != nil {
127+
r.activeViolationsGauge.Record(
128+
ctx,
129+
int64(wp.Status.ActiveViolationCount),
130+
metric.WithAttributes(
131+
attribute.String("policy.name", wp.Name),
132+
attribute.String("k8s.namespace.name", wp.Namespace),
133+
),
134+
)
118135
}
119136
}
120137

0 commit comments

Comments
 (0)