Skip to content

Commit

Permalink
tests: deflakey TestLeaseGrantTimeToLiveExpired
Browse files Browse the repository at this point in the history
The following change can reproduce the issue #17504 by
`TESTCASE=TestLeaseGrantTimeToLiveExpired/PeerAutoTLS make test-e2e`. We
can retry it to reduce flakey possibility, if there is leader change.

```diff
diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go
index afb27c330..dea8459dc 100644
--- a/tests/common/lease_test.go
+++ b/tests/common/lease_test.go
@@ -16,6 +16,7 @@ package common

 import (
        "context"
+       "sync"
        "testing"
        "time"

@@ -139,6 +140,18 @@ func TestLeaseGrantTimeToLiveExpired(t *testing.T) {
                                require.NoError(t, err)
                                require.Equal(t, int64(1), getResp.Count)

+                               var wg sync.WaitGroup
+                               defer wg.Wait()
+
+                               wg.Add(1)
+                               go func() {
+                                       defer wg.Done()
+                                       time.Sleep(1000 * time.Millisecond)
+                                       clus.(interface {
+                                               MoveLeader(ctx context.Context, t testing.TB, i int) error
+                                       }).MoveLeader(ctx, t, (clus.WaitLeader(t)+1)%len(clus.Members()))
+                               }()
+
                                time.Sleep(3 * time.Second)

                                ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{})

```

Signed-off-by: Wei Fu <[email protected]>
  • Loading branch information
fuweid committed Feb 24, 2025
1 parent 498bbc5 commit c81fb87
Showing 1 changed file with 18 additions and 2 deletions.
20 changes: 18 additions & 2 deletions tests/common/lease_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func TestLeaseGrantTimeToLiveExpired(t *testing.T) {

for _, tc := range clusterTestCases() {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config))
defer clus.Close()
Expand All @@ -139,7 +139,23 @@ func TestLeaseGrantTimeToLiveExpired(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(1), getResp.Count)

time.Sleep(3 * time.Second)
// FIXME: When leader changes, old leader steps
// back to follower and ignores the lease revoking.
// The new leader will restart TTL counting. If so,
// we should call time.Sleep again and wait for revoking.
// It can't avoid flakey but reduce flakey possiblility.
for i := 0; i < 3; i++ {
currentLeader := clus.WaitLeader(t)
t.Logf("[%d] current leader index %d", i, currentLeader)

time.Sleep(3 * time.Second)

newLeader := clus.WaitLeader(t)
if newLeader == currentLeader {
break
}
t.Logf("[%d] leader changed, new leader index %d", i, newLeader)
}

ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{})
require.NoError(t, err)
Expand Down

0 comments on commit c81fb87

Please sign in to comment.