fix: excessive records by adding the MaxEvents field (chaos-mesh#4402)

Signed-off-by: Gallardot <[email protected]> Signed-off-by: Yue Yang <[email protected]> Co-authored-by: Yue Yang <[email protected]>
yeya24 · Sep 20, 2024 · 41fb4a2 · 41fb4a2
1 parent bfe8736
commit 41fb4a2
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -67,6 +67,7 @@ For more information and how-to, see [RFC: Keep A Changelog](https://github.com/
 - Fix TTL configuration from environment variables [#4338](https://github.com/chaos-mesh/chaos-mesh/pull/4338)
 - Fix dashboard panic while replacing query namespace with targetNamespace in namespace scoped mode [#4409](https://github.com/chaos-mesh/chaos-mesh/issues/4409)
 - Fix incorrect mmap args for IOChaos [#3680](https://github.com/chaos-mesh/chaos-mesh/issues/3680)
+- Fix excessive records by adding the `MaxEvents` field [#4402](https://github.com/chaos-mesh/chaos-mesh/pull/4402)
 - Fix chaos controller can't find daemonIP over 1000 nodes using endpoints [#4421](https://github.com/chaos-mesh/chaos-mesh/pull/4421)
 - Minor fixes in certificates to make them ArgoCD friendly [#4482](https://github.com/chaos-mesh/chaos-mesh/pull/4482)
 

diff --git a/controllers/common/records/controller.go b/controllers/common/records/controller.go
@@ -29,6 +29,7 @@ import (
 
 	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
 	"github.com/chaos-mesh/chaos-mesh/controllers/chaosimpl/types"
+	"github.com/chaos-mesh/chaos-mesh/controllers/config"
 	"github.com/chaos-mesh/chaos-mesh/controllers/utils/recorder"
 	"github.com/chaos-mesh/chaos-mesh/pkg/selector"
 )
@@ -158,6 +159,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 				// but the retry shouldn't block other resource process
 				idLogger.Error(err, "fail to apply chaos")
 				applyFailedEvent := newRecordEvent(v1alpha1.TypeFailed, v1alpha1.Apply, err.Error())
+				if len(records[index].Events) >= config.ControllerCfg.MaxEvents {
+					records[index].Events = records[index].Events[1:]
+				}
 				records[index].Events = append(records[index].Events, *applyFailedEvent)
 				r.Recorder.Event(obj, recorder.Failed{
 					Activity: "apply chaos",
@@ -172,6 +176,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 			if record.Phase == v1alpha1.Injected {
 				records[index].InjectedCount++
 				applySucceedEvent := newRecordEvent(v1alpha1.TypeSucceeded, v1alpha1.Apply, "")
+				if len(records[index].Events) >= config.ControllerCfg.MaxEvents {
+					records[index].Events = records[index].Events[1:]
+				}
 				records[index].Events = append(records[index].Events, *applySucceedEvent)
 				r.Recorder.Event(obj, recorder.Applied{
 					Id: records[index].Id,
@@ -188,6 +195,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 				// but the retry shouldn't block other resource process
 				idLogger.Error(err, "fail to recover chaos")
 				recoverFailedEvent := newRecordEvent(v1alpha1.TypeFailed, v1alpha1.Recover, err.Error())
+				if len(records[index].Events) >= config.ControllerCfg.MaxEvents {
+					records[index].Events = records[index].Events[1:]
+				}
 				records[index].Events = append(records[index].Events, *recoverFailedEvent)
 				r.Recorder.Event(obj, recorder.Failed{
 					Activity: "recover chaos",
@@ -202,6 +212,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 			if record.Phase == v1alpha1.NotInjected {
 				records[index].RecoveredCount++
 				recoverSucceedEvent := newRecordEvent(v1alpha1.TypeSucceeded, v1alpha1.Recover, "")
+				if len(records[index].Events) >= config.ControllerCfg.MaxEvents {
+					records[index].Events = records[index].Events[1:]
+				}
 				records[index].Events = append(records[index].Events, *recoverSucceedEvent)
 				r.Recorder.Event(obj, recorder.Recovered{
 					Id: records[index].Id,

diff --git a/controllers/utils/chaosdaemon/chaosdaemon.go b/controllers/utils/chaosdaemon/chaosdaemon.go
@@ -67,7 +67,7 @@ func (b *ChaosDaemonClientBuilder) FindDaemonIP(ctx context.Context, pod *v1.Pod
 
 	daemonIP := findIPOnEndpointSlice(&endpointSliceList, nodeName)
 	if len(daemonIP) == 0 {
-		return "", errors.Errorf("cannot find daemonIP on node %s in endpointSliceList %v", nodeName, endpointSliceList)
+		return "", errors.Errorf("cannot find daemonIP on node %s", nodeName)
 	}
 
 	return daemonIP, nil

diff --git a/pkg/config/controller.go b/pkg/config/controller.go
@@ -114,6 +114,8 @@ type ChaosControllerConfig struct {
 	EnabledWebhooks    []string `envconfig:"ENABLED_WEBHOOKS" default:"*"`
 
 	LocalHelmChartPath string `envconfig:"LOCAL_HELM_CHART_PATH" default:""`
+
+	MaxEvents int `envconfig:"MAX_EVENTS" default:"100"`
 }
 
 // EnvironChaosController returns the settings from the environment.