Complete partial downgraded cluster cancellation

Signed-off-by: Chun-Hung Tseng <[email protected]>
etcd-io · Jan 28, 2025 · 6004df8 · 6004df8
1 parent fefbf87
commit 6004df8
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 19 deletions.
diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go
@@ -45,7 +45,7 @@ const (
 	noCancellation CancellationState = iota
 	cancelRightBeforeEnable
 	cancelRightAfterEnable
-	cancelAfterDowngradingSome
+	cancelAfterDowngrading
 )
 
 func TestDowngradeUpgradeClusterOf1(t *testing.T) {
@@ -80,13 +80,13 @@ func TestDowngradeCancellationRightAfterEnablingClusterOf3(t *testing.T) {
 	testDowngradeUpgrade(t, 3, 3, false, cancelRightAfterEnable)
 }
 
-// func TestDowngradeCancellationAfterDowngrading1InClusterOf3(t *testing.T) {
-// 	testDowngradeUpgrade(t, 1, 3, false, cancelAfterDowngradingSome)
-// }
+func TestDowngradeCancellationAfterDowngrading1InClusterOf3(t *testing.T) {
+	testDowngradeUpgrade(t, 1, 3, false, cancelAfterDowngrading)
+}
 
-// func TestDowngradeCancellationAfterDowngrading2InClusterOf3(t *testing.T) {
-// 	testDowngradeUpgrade(t, 2, 3, false, cancelAfterDowngradingSome)
-// }
+func TestDowngradeCancellationAfterDowngrading2InClusterOf3(t *testing.T) {
+	testDowngradeUpgrade(t, 2, 3, false, cancelAfterDowngrading)
+}
 
 func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, triggerSnapshot bool, triggerCancellation CancellationState) {
 	currentEtcdBinary := e2e.BinPath.Etcd
@@ -146,7 +146,7 @@ func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, t
 		t.Logf("Cancelling downgrade before enabling")
 		e2e.DowngradeCancel(t, epc)
 		t.Log("Downgrade cancelled, validating if cluster is in the right state")
-		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
+		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion))
 
 		return // No need to perform downgrading, end the test here
 	}
@@ -155,17 +155,19 @@ func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, t
 		t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)")
 		e2e.DowngradeCancel(t, epc)
 		t.Log("Downgrade cancelled, validating if cluster is in the right state")
-		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr))
+		e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion))
 		return // No need to perform downgrading, end the test here
 	}
 
 	membersToChange := rand.Perm(len(epc.Procs))[:cancellationSize]
-	t.Logf(fmt.Sprintf("Elect members for operations"), zap.Any("members", membersToChange))
+	t.Logf(fmt.Sprintln("Elect members for operations"), zap.Any("members", membersToChange))
 
 	t.Logf("Starting downgrade process to %q", lastVersionStr)
 	err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, membersToChange, currentVersion, lastClusterVersion)
 	require.NoError(t, err)
-	e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")
+	if len(membersToChange) == len(epc.Procs) {
+		e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded")
+	}
 
 	t.Log("Downgrade complete")
 	afterMembers, afterKV := getMembersAndKeys(t, cc)
@@ -176,6 +178,13 @@ func testDowngradeUpgrade(t *testing.T, cancellationSize int, clusterSize int, t
 		t.Log("Waiting health interval to required to make membership changes")
 		time.Sleep(etcdserver.HealthInterval)
 	}
+
+	if triggerCancellation == cancelAfterDowngrading {
+		e2e.DowngradeCancel(t, epc)
+		t.Log("Downgrade cancelled, validating if cluster is in the right state")
+		e2e.ValidateMemberVersions(t, epc, generatePartialCancellationVersions(clusterSize, membersToChange, lastClusterVersion))
+	}
+
 	t.Log("Adding learner to test membership, but avoid breaking quorum")
 	resp, err = cc.MemberAddAsLearner(context.Background(), "fake2", []string{"http://127.0.0.1:1002"})
 	require.NoError(t, err)
@@ -290,16 +299,34 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes
 	return members, kvs
 }
 
-func generateIdenticalVersions(clusterSize int, currentVersion string) []*version.Versions {
+func generateIdenticalVersions(clusterSize int, ver *semver.Version) []*version.Versions {
+	ret := make([]*version.Versions, clusterSize)
+
+	for i := range clusterSize {
+		ret[i] = &version.Versions{
+			Cluster: ver.String(),
+			Server:  ver.String(),
+			Storage: ver.String(),
+		}
+	}
+
+	return ret
+}
+
+func generatePartialCancellationVersions(clusterSize int, membersToChange []int, ver *semver.Version) []*version.Versions {
 	ret := make([]*version.Versions, clusterSize)
 
 	for i := range clusterSize {
 		ret[i] = &version.Versions{
-			Cluster: currentVersion,
-			Server:  currentVersion,
-			Storage: currentVersion,
+			Cluster: ver.String(),
+			Server:  e2e.OffsetMinor(ver, 1).String(),
+			Storage: "",
 		}
 	}
 
+	for i := range membersToChange {
+		ret[membersToChange[i]].Server = ver.String()
+	}
+
 	return ret
 }
diff --git a/tests/framework/e2e/downgrade.go b/tests/framework/e2e/downgrade.go
@@ -44,7 +44,7 @@ func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version)
 	for i := 0; i < len(epc.Procs); i++ {
 		ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
 			Cluster: ver.String(),
-			Server:  offsetMinor(ver, 1).String(),
+			Server:  OffsetMinor(ver, 1).String(),
 			Storage: ver.String(),
 		})
 		AssertProcessLogs(t, epc.Procs[i], "The server is ready to downgrade")
@@ -66,7 +66,7 @@ func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) {
 
 func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error {
 	membersToChange := rand.Perm(len(clus.Procs))[:numberOfMembersToChange]
-	t.Logf(fmt.Sprintf("Elect members for operations"), zap.Any("members", membersToChange))
+	t.Logf(fmt.Sprintln("Elect members for operations"), zap.Any("members", membersToChange))
 
 	return DowngradeUpgradeMembersByID(t, lg, clus, membersToChange, currentVersion, targetVersion)
 }
@@ -148,8 +148,8 @@ func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdPro
 	})
 }
 
-// offsetMinor returns the version with offset from the original minor, with the same major.
-func offsetMinor(v *semver.Version, offset int) *semver.Version {
+// OffsetMinor returns the version with offset from the original minor, with the same major.
+func OffsetMinor(v *semver.Version, offset int) *semver.Version {
 	var minor int64
 	if offset >= 0 {
 		minor = v.Minor + int64(offset)