From a0309011206ae75df138c9bd57d4361ebe141422 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Fri, 9 Jun 2023 15:14:23 +0200 Subject: [PATCH] triggering once is enough --- tests/robustness/failpoints.go | 70 +++++++++++------------- tests/robustness/linearizability_test.go | 6 +- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/tests/robustness/failpoints.go b/tests/robustness/failpoints.go index 29c8dd6116c2..320131976348 100644 --- a/tests/robustness/failpoints.go +++ b/tests/robustness/failpoints.go @@ -531,58 +531,50 @@ type noSpaceFailpoint struct { } func (f noSpaceFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error { - quorumSize := (len(clus.Procs) / 2) + 1 - for i := 0; i < quorumSize; i++ { - member := clus.Procs[i] - if member.IsRunning() { - lg.Info("Setting up gofailpoint", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) - err := member.Failpoints().Setup(ctx, f.Name(), "return") - if err != nil { - lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) + memberIdx := rand.Int() % len(clus.Procs) + member := clus.Procs[memberIdx] + if member.IsRunning() { + lg.Info("Setting up gofailpoint", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) + err := member.Failpoints().Setup(ctx, f.Name(), "return") + if err != nil { + lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) + return err + } + + numRetries := 0 + for { + if numRetries > 5 { + lg.Info("goFailpoint giving up after 5 retries", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) return err } - numRetries := 0 - for { - if numRetries > 5 { - lg.Info("goFailpoint giving up after 5 retries", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) - return err + err = member.Etcdctl().Put(context.Background(), "failpoint_"+member.Config().Name, "", config.PutOptions{}) + if err != nil { + if strings.Contains(err.Error(), rpctypes.ErrGRPCNoSpace.Error()) { + lg.Info("goFailpoint triggered successfully", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) + break } - err = member.Etcdctl().Put(context.Background(), "failpoint_"+member.Config().Name, "", config.PutOptions{}) - if err != nil { - if strings.Contains(err.Error(), rpctypes.ErrGRPCNoSpace.Error()) { - lg.Info("goFailpoint triggered successfully", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) - break - } - - lg.Info("goFailpoint received unrelated error, retrying", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) - time.Sleep(1 * time.Second) - numRetries++ - continue - } + lg.Info("goFailpoint received unrelated error, retrying", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) + time.Sleep(1 * time.Second) + numRetries++ + continue } } } time.Sleep(f.alarmDuration) - for i := 0; i < quorumSize; i++ { - member := clus.Procs[i] - lg.Info("goFailpoint starting to disable", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) - - err := member.Failpoints().Disable(ctx, f.Name()) - if err != nil { - lg.Info("goFailpoint disable failed", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) - return err - } - - lg.Info("goFailpoint disabled successfully", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) + lg.Info("goFailpoint starting to disable", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) + err := member.Failpoints().Disable(ctx, f.Name()) + if err != nil { + lg.Info("goFailpoint disable failed", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) + return err } - // disarming can be done once at the end, clearing all alarms - member := clus.Procs[0] - _, err := member.Etcdctl().AlarmDisarm(ctx, nil) + lg.Info("goFailpoint disabled successfully", zap.String("failpoint", f.Name()), zap.String("target", member.Config().Name)) + + _, err = member.Etcdctl().AlarmDisarm(ctx, nil) if err != nil { lg.Info("goFailpoint alarm disarm failed", zap.String("failpoint", f.Name()), zap.Error(err), zap.String("target", member.Config().Name)) return err diff --git a/tests/robustness/linearizability_test.go b/tests/robustness/linearizability_test.go index a843167ab191..04046f9be85f 100644 --- a/tests/robustness/linearizability_test.go +++ b/tests/robustness/linearizability_test.go @@ -104,14 +104,10 @@ func TestRobustness(t *testing.T) { scenarios = append(scenarios, testScenario{ name: "IssueXYZ", failpoint: PutReturnErrNoSpace, - traffic: traffic.KubernetesTraffic, + traffic: traffic.LowTraffic, clusterOpts: []e2e.EPClusterOption{ e2e.WithIsPeerTLS(true), - e2e.WithSnapshotCount(100), - e2e.WithPeerProxy(true), e2e.WithGoFailEnabled(true), - e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), - e2e.WithLogLevel("debug"), }, }) if v.Compare(version.V3_5) >= 0 {