From e25eca27e35fe86b707939d30949eb1ee4d1575f Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 21 Jan 2025 15:09:45 +0000 Subject: [PATCH 1/3] Add downgrade cancellation e2e tests Signed-off-by: Chun-Hung Tseng Co-authored-by: Benjamin Wang Signed-off-by: Chun-Hung Tseng --- etcdctl/README.md | 2 +- tests/e2e/cluster_downgrade_test.go | 60 ++++++++++++++++++++++++++--- tests/framework/e2e/downgrade.go | 20 ++++++++++ tests/framework/e2e/etcdctl.go | 5 +++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/etcdctl/README.md b/etcdctl/README.md index 40e93c4d1c0..3589f4d699d 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -1119,7 +1119,7 @@ DOWNGRADE ENABLE starts a downgrade action to cluster. Downgrade enable success, cluster version 3.6 ``` -### DOWNGRADE CANCEL \ +### DOWNGRADE CANCEL DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster. diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 086b80894d5..5779e0547d6 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -37,23 +37,47 @@ import ( "go.etcd.io/etcd/tests/v3/framework/e2e" ) +type CancellationState int + +const ( + noCancellation CancellationState = iota + cancelRightBeforeEnable + cancelRightAfterEnable +) + func TestDowngradeUpgradeClusterOf1(t *testing.T) { - testDowngradeUpgrade(t, 1, false) + testDowngradeUpgrade(t, 1, false, noCancellation) } func TestDowngradeUpgradeClusterOf3(t *testing.T) { - testDowngradeUpgrade(t, 3, false) + testDowngradeUpgrade(t, 3, false, noCancellation) } func TestDowngradeUpgradeClusterOf1WithSnapshot(t *testing.T) { - testDowngradeUpgrade(t, 1, true) + testDowngradeUpgrade(t, 1, true, noCancellation) } func TestDowngradeUpgradeClusterOf3WithSnapshot(t *testing.T) { - testDowngradeUpgrade(t, 3, true) + testDowngradeUpgrade(t, 3, true, noCancellation) +} + +func TestDowngradeCancellationWithoutEnablingClusterOf1(t *testing.T) { + testDowngradeUpgrade(t, 1, false, cancelRightBeforeEnable) +} + +func TestDowngradeCancellationRightAfterEnablingClusterOf1(t *testing.T) { + testDowngradeUpgrade(t, 1, false, cancelRightAfterEnable) +} + +func TestDowngradeCancellationWithoutEnablingClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 3, false, cancelRightBeforeEnable) +} + +func TestDowngradeCancellationRightAfterEnablingClusterOf3(t *testing.T) { + testDowngradeUpgrade(t, 3, false, cancelRightAfterEnable) } -func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) { +func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool, triggerCancellation CancellationState) { currentEtcdBinary := e2e.BinPath.Etcd lastReleaseBinary := e2e.BinPath.EtcdLastRelease if !fileutil.Exist(lastReleaseBinary) { @@ -107,7 +131,18 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool) { require.NoError(t, err) beforeMembers, beforeKV := getMembersAndKeys(t, cc) + if triggerCancellation == cancelRightBeforeEnable { + t.Logf("Cancelling downgrade before enabling") + e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + return // No need to perform downgrading, end the test here + } e2e.DowngradeEnable(t, epc, lastVersion) + if triggerCancellation == cancelRightAfterEnable { + t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)") + e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + return // No need to perform downgrading, end the test here + } + t.Logf("Starting downgrade process to %q", lastVersionStr) e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion) e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded") @@ -233,3 +268,18 @@ func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListRes return members, kvs } + +func generateIdenticalVersions(clusterSize int, currentVersion string) []*version.Versions { + ret := make([]*version.Versions, clusterSize) + + for i := range clusterSize { + ret[i] = &version.Versions{ + Cluster: currentVersion, + Server: currentVersion, + Storage: currentVersion, + } + + } + + return ret +} diff --git a/tests/framework/e2e/downgrade.go b/tests/framework/e2e/downgrade.go index 60a45adedae..3f771f1bcc1 100644 --- a/tests/framework/e2e/downgrade.go +++ b/tests/framework/e2e/downgrade.go @@ -53,6 +53,26 @@ func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version) t.Log("Cluster is ready for downgrade") } +func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster, versions []*version.Versions) { + t.Logf("etcdctl downgrade cancel") + c := epc.Etcdctl() + testutils.ExecuteWithTimeout(t, 20*time.Second, func() { + err := c.DowngradeCancel(context.TODO()) + require.NoError(t, err) + }) + + t.Log("Downgrade cancelled, validating if cluster is in the right state") + for i := 0; i < len(epc.Procs); i++ { + ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ + Cluster: versions[i].Cluster, + Server: versions[i].Server, + Storage: versions[i].Storage, + }) + } + + t.Log("Cluster downgrade cancellation is completed") +} + func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, currentVersion, targetVersion *semver.Version) error { if lg == nil { lg = clus.lg diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index d0c8dc14c72..9235ab28d85 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -86,6 +86,11 @@ func (ctl *EtcdctlV3) DowngradeEnable(ctx context.Context, version string) error return err } +func (ctl *EtcdctlV3) DowngradeCancel(ctx context.Context) error { + _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "cancel"), nil, expect.ExpectedResponse{Value: "Downgrade cancel success"}) + return err +} + func (ctl *EtcdctlV3) Get(ctx context.Context, key string, o config.GetOptions) (*clientv3.GetResponse, error) { resp := clientv3.GetResponse{} var args []string From a3c072c3e87a553f611ac9511b9f94ff3a43c002 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 21 Jan 2025 21:28:41 +0000 Subject: [PATCH 2/3] Add missing error checking after calling DowngradeUpgradeMembers Signed-off-by: Chun-Hung Tseng Signed-off-by: Chun-Hung Tseng --- tests/e2e/cluster_downgrade_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index 5779e0547d6..f71cae5b535 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -144,7 +144,8 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool, t } t.Logf("Starting downgrade process to %q", lastVersionStr) - e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion) + err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), currentVersion, lastClusterVersion) + require.NoError(t, err) e2e.AssertProcessLogs(t, leader(t, epc), "the cluster has been downgraded") t.Log("Downgrade complete") @@ -170,7 +171,8 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool, t beforeMembers, beforeKV = getMembersAndKeys(t, cc) t.Logf("Starting upgrade process to %q", currentVersionStr) - e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), lastClusterVersion, currentVersion) + err = e2e.DowngradeUpgradeMembers(t, nil, epc, len(epc.Procs), lastClusterVersion, currentVersion) + require.NoError(t, err) t.Log("Upgrade complete") afterMembers, afterKV = getMembersAndKeys(t, cc) @@ -278,7 +280,6 @@ func generateIdenticalVersions(clusterSize int, currentVersion string) []*versio Server: currentVersion, Storage: currentVersion, } - } return ret From 2e41777edb1180b279d66d3cb7570d0611682321 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Tue, 28 Jan 2025 12:47:53 +0000 Subject: [PATCH 3/3] Introduce ValidateMemberVersions Signed-off-by: Chun-Hung Tseng --- tests/e2e/cluster_downgrade_test.go | 9 +++++++-- tests/framework/e2e/downgrade.go | 18 ++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/e2e/cluster_downgrade_test.go b/tests/e2e/cluster_downgrade_test.go index f71cae5b535..341f8528249 100644 --- a/tests/e2e/cluster_downgrade_test.go +++ b/tests/e2e/cluster_downgrade_test.go @@ -133,13 +133,18 @@ func testDowngradeUpgrade(t *testing.T, clusterSize int, triggerSnapshot bool, t if triggerCancellation == cancelRightBeforeEnable { t.Logf("Cancelling downgrade before enabling") - e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + e2e.DowngradeCancel(t, epc) + t.Log("Downgrade cancelled, validating if cluster is in the right state") + e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + return // No need to perform downgrading, end the test here } e2e.DowngradeEnable(t, epc, lastVersion) if triggerCancellation == cancelRightAfterEnable { t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)") - e2e.DowngradeCancel(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) + e2e.DowngradeCancel(t, epc) + t.Log("Downgrade cancelled, validating if cluster is in the right state") + e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersionStr)) return // No need to perform downgrading, end the test here } diff --git a/tests/framework/e2e/downgrade.go b/tests/framework/e2e/downgrade.go index 3f771f1bcc1..52a7c4c8744 100644 --- a/tests/framework/e2e/downgrade.go +++ b/tests/framework/e2e/downgrade.go @@ -53,7 +53,7 @@ func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version) t.Log("Cluster is ready for downgrade") } -func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster, versions []*version.Versions) { +func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) { t.Logf("etcdctl downgrade cancel") c := epc.Etcdctl() testutils.ExecuteWithTimeout(t, 20*time.Second, func() { @@ -61,15 +61,6 @@ func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster, versions []*version. require.NoError(t, err) }) - t.Log("Downgrade cancelled, validating if cluster is in the right state") - for i := 0; i < len(epc.Procs); i++ { - ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ - Cluster: versions[i].Cluster, - Server: versions[i].Server, - Storage: versions[i].Storage, - }) - } - t.Log("Cluster downgrade cancellation is completed") } @@ -124,6 +115,13 @@ func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessClus return nil } +func ValidateMemberVersions(t *testing.T, epc *EtcdProcessCluster, expect []*version.Versions) { + for i := 0; i < len(epc.Procs); i++ { + ValidateVersion(t, epc.Cfg, epc.Procs[i], *expect[i]) + } + t.Log("Cluster member version validation after downgrade cancellation is completed") +} + func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdProcess, expect version.Versions) { testutils.ExecuteWithTimeout(t, 30*time.Second, func() { for {