From e5ba13ede911b797cce5a5ed0d4883523fbedf1d Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 15 May 2023 17:00:35 -0700
Subject: [PATCH 01/44] Update ValidateConfigurationChange to nil check for
 v2store Part of v2 deprecation changes

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index 31fb088f8db..95cd05bec28 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -305,8 +305,13 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 // ValidateConfigurationChange takes a proposed ConfChange and
 // ensures that it is still valid.
 func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
-	// TODO: this must be switched to backend as well.
-	membersMap, removedMap := membersFromStore(c.lg, c.v2store)
+	var membersMap map[types.ID]*Member
+	var removedMap map[types.ID]bool
+	if c.v2store != nil {
+		membersMap, removedMap = membersFromStore(c.lg, c.v2store)
+	} else {
+		membersMap, removedMap = c.be.MustReadMembersFromBackend()
+	}
 	id := types.ID(cc.NodeID)
 	if removedMap[id] {
 		return ErrIDRemoved

From 71c0a65f350d100388bc9df755129313184c40f0 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 16 May 2023 11:39:05 -0700
Subject: [PATCH 02/44] v2d store RaftCluster membership info in v2 during
 snapshot

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 14 ++++++++++++++
 server/etcdserver/server.go                 |  6 +++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index 95cd05bec28..5287fd91866 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -860,3 +860,17 @@ func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearner
 
 	return nil
 }
+
+func (c *RaftCluster) Store(store v2store.Store) {
+	c.Lock()
+	defer c.Unlock()
+	for _, m := range c.members {
+		mustSaveMemberToStore(c.lg, store, m)
+	}
+	for id, _ := range c.removed {
+		mustDeleteMemberFromStore(c.lg, store, id)
+	}
+	if c.version != nil {
+		mustSaveClusterVersionToStore(c.lg, store, c.version)
+	}
+}
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 13fc5f5b360..51c819d89a1 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -2052,7 +2052,6 @@ func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.Con
 
 // TODO: non-blocking snapshot
 func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
-	clone := s.v2store.Clone()
 	// commit kv to write metadata (for example: consistent index) to disk.
 	//
 	// This guarantees that Backend's consistent_index is >= index of last snapshot.
@@ -2066,8 +2065,9 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 
 	s.GoAttach(func() {
 		lg := s.Logger()
-
-		d, err := clone.SaveNoCopy()
+		st := v2store.New(StoreClusterPrefix, StoreKeysPrefix)
+		s.cluster.Store(st)
+		d, err := st.SaveNoCopy()
 		// TODO: current store will never fail to do a snapshot
 		// what should we do if the store might fail?
 		if err != nil {

From a749ef407638ed8676b0aaca7b2c806aa60ca010 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 16 May 2023 16:20:59 -0700
Subject: [PATCH 03/44] updating based on test failure

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/storev2.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/server/etcdserver/api/membership/storev2.go b/server/etcdserver/api/membership/storev2.go
index d428cb66e22..209f7d9726c 100644
--- a/server/etcdserver/api/membership/storev2.go
+++ b/server/etcdserver/api/membership/storev2.go
@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path"
+	"strings"
 
 	"go.etcd.io/etcd/client/pkg/v3/types"
 
@@ -95,11 +96,13 @@ func mustSaveMemberToStore(lg *zap.Logger, s v2store.Store, m *Member) {
 
 func mustDeleteMemberFromStore(lg *zap.Logger, s v2store.Store, id types.ID) {
 	if _, err := s.Delete(MemberStoreKey(id), true, true); err != nil {
-		lg.Panic(
-			"failed to delete member from store",
-			zap.String("path", MemberStoreKey(id)),
-			zap.Error(err),
-		)
+		if !strings.Contains(err.Error(), "Key not found") {
+			lg.Panic(
+				"failed to delete member from store",
+				zap.String("path", MemberStoreKey(id)),
+				zap.Error(err),
+			)
+		}
 	}
 	if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil {
 		lg.Panic(

From 9f66d43e8f6ab240844e1a246265113b40d674af Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 17 May 2023 03:01:37 +0000
Subject: [PATCH 04/44] updating to address test failure

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 2bf113505f4..8ccd51fff86 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1062,17 +1062,22 @@ func TestSnapshot(t *testing.T) {
 		gaction, _ := st.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
-		if len(gaction) != 2 {
-			t.Errorf("len(action) = %d, want 2", len(gaction))
-		}
-		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
-			t.Errorf("action = %s, want Clone", gaction[0])
-		}
-		if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
-			t.Errorf("action = %s, want SaveNoCopy", gaction[1])
+		//snapshot will be stored in brand new v2store constructed during snapshot
+		//eventually EtcdServer struct will not have v2store field
+		if len(gaction) != 0 {
+			t.Errorf("len(action) = %d, want 0", len(gaction))
 		}
+		//if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
+		//	t.Errorf("action = %s, want Clone", gaction[0])
+		//}
+		//if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
+		//	t.Errorf("action = %s, want SaveNoCopy", gaction[1])
+		//}
 	}()
 
+	lg := zaptest.NewLogger(t)
+	cl := membership.NewCluster(lg)
+	srv.cluster = cl
 	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
 	<-ch
 	<-ch
@@ -1202,6 +1207,9 @@ func TestTriggerSnap(t *testing.T) {
 
 	srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{})
 	srv.be = be
+	lg := zaptest.NewLogger(t)
+	cl := membership.NewCluster(lg)
+	srv.cluster = cl
 
 	srv.start()
 

From 39a7837954de72fd800ec452e05cd829067bda70 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 17 May 2023 04:41:10 +0000
Subject: [PATCH 05/44] refactor v2 deprecation change for snapshot

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go      |  9 ++++-
 server/etcdserver/server_test.go | 69 +++++++++++++++++++++++++-------
 2 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 51c819d89a1..86f8ecca2f0 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -2065,8 +2065,13 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 
 	s.GoAttach(func() {
 		lg := s.Logger()
-		st := v2store.New(StoreClusterPrefix, StoreKeysPrefix)
-		s.cluster.Store(st)
+		var st v2store.Store
+		if s.v2store == nil {
+			st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
+			s.cluster.Store(st)
+		} else {
+			st = s.v2store.Clone()
+		}
 		d, err := st.SaveNoCopy()
 		// TODO: current store will never fail to do a snapshot
 		// what should we do if the store might fail?
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 8ccd51fff86..8a6e150378a 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1062,25 +1062,69 @@ func TestSnapshot(t *testing.T) {
 		gaction, _ := st.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
-		//snapshot will be stored in brand new v2store constructed during snapshot
-		//eventually EtcdServer struct will not have v2store field
-		if len(gaction) != 0 {
-			t.Errorf("len(action) = %d, want 0", len(gaction))
+		if len(gaction) != 2 {
+			t.Errorf("len(action) = %d, want 2", len(gaction))
+		}
+		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
+			t.Errorf("action = %s, want Clone", gaction[0])
+		}
+		if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
+			t.Errorf("action = %s, want SaveNoCopy", gaction[1])
 		}
-		//if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
-		//	t.Errorf("action = %s, want Clone", gaction[0])
-		//}
-		//if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
-		//	t.Errorf("action = %s, want SaveNoCopy", gaction[1])
-		//}
 	}()
 
+	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
+	<-ch
+	<-ch
+}
+
+// TestSnapshotNoV2store should create snapshot using new v2store
+func TestSnapshotNoV2store(t *testing.T) {
+	be, _ := betesting.NewDefaultTmpBackend(t)
+
+	s := raft.NewMemoryStorage()
+	s.Append([]raftpb.Entry{{Index: 1}})
+	p := mockstorage.NewStorageRecorderStream("")
+	r := newRaftNode(raftNodeConfig{
+		lg:          zaptest.NewLogger(t),
+		Node:        newNodeNop(),
+		raftStorage: s,
+		storage:     p,
+	})
+	srv := &EtcdServer{
+		lgMu:         new(sync.RWMutex),
+		lg:           zaptest.NewLogger(t),
+		r:            *r,
+		consistIndex: cindex.NewConsistentIndex(be),
+	}
+	srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{})
+	srv.be = be
 	lg := zaptest.NewLogger(t)
+	// TODO use a mock raft cluster implementation to validate the v2store
 	cl := membership.NewCluster(lg)
 	srv.cluster = cl
+
+	ch := make(chan struct{}, 1)
+
+	go func() {
+		gaction, _ := p.Wait(2)
+		defer func() { ch <- struct{}{} }()
+
+		if len(gaction) != 2 {
+			t.Errorf("len(action) = %d, want 2", len(gaction))
+			return
+		}
+		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "SaveSnap"}) {
+			t.Errorf("action = %s, want SaveSnap", gaction[0])
+		}
+
+		if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "Release"}) {
+			t.Errorf("action = %s, want Release", gaction[1])
+		}
+	}()
+
 	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
 	<-ch
-	<-ch
 }
 
 // TestSnapshotOrdering ensures raft persists snapshot onto disk before
@@ -1207,9 +1251,6 @@ func TestTriggerSnap(t *testing.T) {
 
 	srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{})
 	srv.be = be
-	lg := zaptest.NewLogger(t)
-	cl := membership.NewCluster(lg)
-	srv.cluster = cl
 
 	srv.start()
 

From 6798d775e50e39e5bd2a275ff07b83c05525579b Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 17 May 2023 23:06:52 +0000
Subject: [PATCH 06/44] persist "attributes" of member to new v2store

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index 5287fd91866..fdf1e0ac445 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -866,6 +866,9 @@ func (c *RaftCluster) Store(store v2store.Store) {
 	defer c.Unlock()
 	for _, m := range c.members {
 		mustSaveMemberToStore(c.lg, store, m)
+		if m.ClientURLs != nil {
+			mustUpdateMemberAttrInStore(c.lg, store, m)
+		}
 	}
 	for id, _ := range c.removed {
 		mustDeleteMemberFromStore(c.lg, store, id)

From 11fa7d30bbbd417277731d80f8fb270d6bb43c6b Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Thu, 18 May 2023 03:34:24 +0000
Subject: [PATCH 07/44] Adding unit test

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 .../etcdserver/api/membership/cluster_test.go | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go
index ce98472df7b..0b33720612e 100644
--- a/server/etcdserver/api/membership/cluster_test.go
+++ b/server/etcdserver/api/membership/cluster_test.go
@@ -21,6 +21,8 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/coreos/go-semver/semver"
+	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
@@ -975,3 +977,42 @@ func TestIsReadyToPromoteMember(t *testing.T) {
 		}
 	}
 }
+
+// TestMembershipStore tests code path used by snapshot
+func TestMembershipStore(t *testing.T) {
+	name := "etcd"
+	clientURLs := []string{"http://127.0.0.1:4001"}
+	tests := []struct {
+		mems    []*Member
+		removed map[types.ID]bool
+	}{
+		// update attributes of existing member
+		{
+			[]*Member{
+				newTestMember(2, nil, name, clientURLs),
+			},
+			map[types.ID]bool{types.ID(1): true},
+		},
+	}
+	for i, tt := range tests {
+		c := newTestCluster(t, tt.mems)
+		c.removed = tt.removed
+
+		//snapshot
+		st := v2store.New("/0", "/1")
+		c.Store(st)
+		d, _ := st.SaveNoCopy()
+
+		//recover from snapshot
+		rst := v2store.New("/0", "/1")
+		rst.Recovery(d)
+		rc := &RaftCluster{lg: zaptest.NewLogger(t), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
+		rc.SetStore(rst)
+		rc.Recover(func(lg *zap.Logger, v *semver.Version) { return })
+
+		//membership should match
+		if g := rc.Members(); !reflect.DeepEqual(g, tt.mems) {
+			t.Errorf("#%d: members = %+v, want %+v", i, g, tt.mems)
+		}
+	}
+}

From f55890c655678e59e1d3639087736dc205d1e84c Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 17 May 2023 03:01:37 +0000
Subject: [PATCH 08/44] update v2 deprecation test

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go           |  8 ++---
 server/etcdserver/server_test.go      | 40 +++++++++++++++---------
 tests/e2e/v2store_deprecation_test.go | 44 ++++++++++++++++++++++++---
 3 files changed, 66 insertions(+), 26 deletions(-)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 86f8ecca2f0..76509d088ca 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -2066,12 +2066,8 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 	s.GoAttach(func() {
 		lg := s.Logger()
 		var st v2store.Store
-		if s.v2store == nil {
-			st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
-			s.cluster.Store(st)
-		} else {
-			st = s.v2store.Clone()
-		}
+		st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
+		s.cluster.Store(st)
 		d, err := st.SaveNoCopy()
 		// TODO: current store will never fail to do a snapshot
 		// what should we do if the store might fail?
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 8a6e150378a..274b9dbfdfb 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1038,8 +1038,11 @@ func TestSnapshot(t *testing.T) {
 	}
 	srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{})
 	srv.be = be
+	lg := zaptest.NewLogger(t)
+	cl := membership.NewCluster(lg)
+	srv.cluster = cl
 
-	ch := make(chan struct{}, 2)
+	ch := make(chan struct{}, 1)
 
 	go func() {
 		gaction, _ := p.Wait(2)
@@ -1058,24 +1061,28 @@ func TestSnapshot(t *testing.T) {
 		}
 	}()
 
-	go func() {
-		gaction, _ := st.Wait(2)
-		defer func() { ch <- struct{}{} }()
+	/*
+		//snapshot will be stored in brand new v2store constructed during snapshot
+		//eventually EtcdServer struct will not have v2store field
+		go func() {
+			gaction, _ := st.Wait(2)
+			defer func() { ch <- struct{}{} }()
 
-		if len(gaction) != 2 {
-			t.Errorf("len(action) = %d, want 2", len(gaction))
-		}
-		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
-			t.Errorf("action = %s, want Clone", gaction[0])
-		}
-		if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
-			t.Errorf("action = %s, want SaveNoCopy", gaction[1])
-		}
-	}()
+			if len(gaction) != 2 {
+				t.Errorf("len(action) = %d, want 0", len(gaction))
+			}
+			//if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
+			//	t.Errorf("action = %s, want Clone", gaction[0])
+			//}
+			//if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
+			//	t.Errorf("action = %s, want SaveNoCopy", gaction[1])
+			//}
+		}()
+	*/
 
 	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
 	<-ch
-	<-ch
+	//<-ch
 }
 
 // TestSnapshotNoV2store should create snapshot using new v2store
@@ -1251,6 +1258,9 @@ func TestTriggerSnap(t *testing.T) {
 
 	srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{})
 	srv.be = be
+	lg := zaptest.NewLogger(t)
+	cl := membership.NewCluster(lg)
+	srv.cluster = cl
 
 	srv.start()
 
diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go
index cb4dea9cc8a..c646605b239 100644
--- a/tests/e2e/v2store_deprecation_test.go
+++ b/tests/e2e/v2store_deprecation_test.go
@@ -17,16 +17,21 @@ package e2e
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"reflect"
 	"sort"
 	"strings"
 	"testing"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/server/v3/etcdserver"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/tests/v3/framework/config"
@@ -128,14 +133,14 @@ func TestV2DeprecationSnapshotMatches(t *testing.T) {
 
 	assertSnapshotsMatch(t, oldMemberDataDir, newMemberDataDir, func(data []byte) []byte {
 		// Patch cluster version
-		data = bytes.Replace(data, []byte("3.5.0"), []byte("X.X.X"), -1)
-		data = bytes.Replace(data, []byte("3.6.0"), []byte("X.X.X"), -1)
+		//data = bytes.Replace(data, []byte("3.5.0"), []byte("X.X.X"), -1)
+		//data = bytes.Replace(data, []byte("3.6.0"), []byte("X.X.X"), -1)
 		// Patch members ids
 		for i, mid := range members1 {
-			data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("member%d", i+1)), -1)
+			data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("%d", i+1)), -1)
 		}
 		for i, mid := range members2 {
-			data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("member%d", i+1)), -1)
+			data = bytes.Replace(data, []byte(fmt.Sprintf("%x", mid)), []byte(fmt.Sprintf("%d", i+1)), -1)
 		}
 		return data
 	})
@@ -255,12 +260,41 @@ func assertSnapshotsMatch(t testing.TB, firstDataDir, secondDataDir string, patc
 		if err != nil {
 			t.Fatal(err)
 		}
-		assert.Equal(t, openSnap(patch(firstSnapshot.Data)), openSnap(patch(secondSnapshot.Data)))
+		//assert.Equal(t, openSnap(patch(firstSnapshot.Data)), openSnap(patch(secondSnapshot.Data)))
+		assertMembershipEqual(t, openSnap(patch(firstSnapshot.Data)), openSnap(patch(secondSnapshot.Data)))
 	}
 }
 
 func openSnap(data []byte) v2store.Store {
 	st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
 	st.Recovery(data)
+	//TODO remove the printing.
+	prettyPrintJson(data)
 	return st
 }
+
+func assertMembershipEqual(t testing.TB, firstStore v2store.Store, secondStore v2store.Store) {
+	rc1 := membership.NewCluster(zaptest.NewLogger(t))
+	rc1.SetStore(firstStore)
+	rc1.Recover(func(lg *zap.Logger, v *semver.Version) { return })
+
+	rc2 := membership.NewCluster(zaptest.NewLogger(t))
+	rc2.SetStore(secondStore)
+	rc2.Recover(func(lg *zap.Logger, v *semver.Version) { return })
+
+	//membership should match
+	if g := rc1.Members(); !reflect.DeepEqual(g, rc2.Members()) {
+		fmt.Printf("memberids_from_last_version = %+v, member_ids_from_current_version = %+v\n", rc1.MemberIDs(), rc2.MemberIDs())
+		t.Errorf("members_from_last_version_snapshot = %+v, members_from_current_version_snapshot %+v", rc1.Members(), rc2.Members())
+	}
+}
+
+func prettyPrintJson(jsonData []byte) {
+	var out bytes.Buffer
+	err := json.Indent(&out, jsonData, "", " ")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(string(out.Bytes()))
+}

From e467de9d68b828c21d2a976a15243f9775954a58 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Thu, 18 May 2023 22:22:21 +0000
Subject: [PATCH 09/44] update test

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 38 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 274b9dbfdfb..897f27db416 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1042,7 +1042,7 @@ func TestSnapshot(t *testing.T) {
 	cl := membership.NewCluster(lg)
 	srv.cluster = cl
 
-	ch := make(chan struct{}, 1)
+	ch := make(chan struct{}, 2)
 
 	go func() {
 		gaction, _ := p.Wait(2)
@@ -1061,28 +1061,28 @@ func TestSnapshot(t *testing.T) {
 		}
 	}()
 
-	/*
-		//snapshot will be stored in brand new v2store constructed during snapshot
-		//eventually EtcdServer struct will not have v2store field
-		go func() {
-			gaction, _ := st.Wait(2)
-			defer func() { ch <- struct{}{} }()
+	go func() {
+		gaction, _ := st.Wait(2)
+		defer func() { ch <- struct{}{} }()
 
-			if len(gaction) != 2 {
-				t.Errorf("len(action) = %d, want 0", len(gaction))
-			}
-			//if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
-			//	t.Errorf("action = %s, want Clone", gaction[0])
-			//}
-			//if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
-			//	t.Errorf("action = %s, want SaveNoCopy", gaction[1])
-			//}
-		}()
-	*/
+		//v2 deprecation:
+		//snapshot will be stored in brand new v2store constructed during snapshot
+		//there will be no action on the v2store that is part of EtcdServer
+		//Eventually EtcdServer will not have v2store
+		if len(gaction) != 0 {
+			t.Errorf("len(action) = %d, want 0", len(gaction))
+		}
+		//if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "Clone"}) {
+		//	t.Errorf("action = %s, want Clone", gaction[0])
+		//}
+		//if !reflect.DeepEqual(gaction[1], testutil.Action{Name: "SaveNoCopy"}) {
+		//	t.Errorf("action = %s, want SaveNoCopy", gaction[1])
+		//}
+	}()
 
 	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
 	<-ch
-	//<-ch
+	<-ch
 }
 
 // TestSnapshotNoV2store should create snapshot using new v2store

From 7a8bb94b60e55b101cb0dd25977f2a602360c8f7 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 23 May 2023 17:38:01 +0000
Subject: [PATCH 10/44] Adding test for recovering old version from v2 snapshot
 taken by new version

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 tests/e2e/v2store_deprecation_test.go | 61 +++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go
index c646605b239..9421f9337c9 100644
--- a/tests/e2e/v2store_deprecation_test.go
+++ b/tests/e2e/v2store_deprecation_test.go
@@ -29,6 +29,7 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
+	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
@@ -187,6 +188,66 @@ func TestV2DeprecationSnapshotRecover(t *testing.T) {
 	assert.NoError(t, epc.Close())
 }
 
+func TestV2DeprecationSnapshotRecoverOldVersion(t *testing.T) {
+	e2e.BeforeTest(t)
+	dataDir := t.TempDir()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
+		t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease)
+	}
+	epc := runEtcdAndCreateSnapshot(t, e2e.CurrentVersion, dataDir, 10)
+	lastVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease)
+	lastVersionStr := lastVersion.String()
+
+	lastClusterVersion := semver.New(lastVersionStr)
+	lastClusterVersion.Patch = 0
+	lastClusterVersionStr := lastClusterVersion.String()
+
+	t.Logf("etcdctl downgrade enable %s", lastVersionStr)
+	downgradeEnable(t, epc, lastVersion)
+
+	t.Log("Downgrade enabled, validating if cluster is ready for downgrade")
+	for i := 0; i < len(epc.Procs); i++ {
+		validateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{
+			Cluster: lastClusterVersionStr,
+			Server:  version.Version,
+			Storage: lastClusterVersionStr,
+		})
+		e2e.AssertProcessLogs(t, epc.Procs[i], "The server is ready to downgrade")
+	}
+
+	cc, err := e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
+	assert.NoError(t, err)
+
+	lastReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
+	assert.NoError(t, err)
+
+	lastReleaseMemberListResponse, err := cc.MemberList(ctx, false)
+	assert.NoError(t, err)
+
+	assert.NoError(t, epc.Close())
+	cfg := e2e.ConfigStandalone(*e2e.NewConfig(
+		e2e.WithVersion(e2e.LastVersion),
+		e2e.WithDataDirPath(dataDir),
+	))
+	epc, err = e2e.NewEtcdProcessCluster(context.TODO(), t, e2e.WithConfig(cfg))
+	assert.NoError(t, err)
+
+	cc, err = e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
+	assert.NoError(t, err)
+	currentReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
+	assert.NoError(t, err)
+
+	currentReleaseMemberListResponse, err := cc.MemberList(ctx, false)
+	assert.NoError(t, err)
+
+	assert.Equal(t, lastReleaseGetResponse.Kvs, currentReleaseGetResponse.Kvs)
+	assert.Equal(t, lastReleaseMemberListResponse.Members, currentReleaseMemberListResponse.Members)
+	assert.NoError(t, epc.Close())
+}
+
 func runEtcdAndCreateSnapshot(t testing.TB, serverVersion e2e.ClusterVersion, dataDir string, snapshotCount uint64) *e2e.EtcdProcessCluster {
 	cfg := e2e.ConfigStandalone(*e2e.NewConfig(
 		e2e.WithVersion(serverVersion),

From f41ed2e5a2ea4d7a1fef59d68cdf7d6e60354e72 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 23 May 2023 18:52:23 +0000
Subject: [PATCH 11/44] Add downgrade test that creates snapshots and keys

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go |  6 ++++
 server/etcdserver/bootstrap.go              |  1 +
 tests/e2e/v2store_deprecation_test.go       | 40 ++++++++++++---------
 3 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index fdf1e0ac445..5e272cf09ed 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -869,6 +869,12 @@ func (c *RaftCluster) Store(store v2store.Store) {
 		if m.ClientURLs != nil {
 			mustUpdateMemberAttrInStore(c.lg, store, m)
 		}
+		c.lg.Info(
+			"snapshot storing member",
+			zap.String("id", m.ID.String()),
+			zap.Strings("peer-urls", m.PeerURLs),
+			zap.Bool("is-learner", m.IsLearner),
+		)
 	}
 	for id, _ := range c.removed {
 		mustDeleteMemberFromStore(c.lg, store, id)
diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index e416fd079c2..570b6e63fd9 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -221,6 +221,7 @@ func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, st v2store.Store, s
 	// TODO(serathius): Implement schema setup in fresh storage
 	var snapshot *raftpb.Snapshot
 	if haveWAL {
+		//geetasg TODO input st is empty - remove this step for v2deprecation and replace with recoverying from be
 		snapshot, be, err = recoverSnapshot(cfg, st, be, beExist, beHooks, ci, ss)
 		if err != nil {
 			return nil, err
diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go
index 9421f9337c9..91582495d00 100644
--- a/tests/e2e/v2store_deprecation_test.go
+++ b/tests/e2e/v2store_deprecation_test.go
@@ -197,14 +197,14 @@ func TestV2DeprecationSnapshotRecoverOldVersion(t *testing.T) {
 	if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) {
 		t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease)
 	}
-	epc := runEtcdAndCreateSnapshot(t, e2e.CurrentVersion, dataDir, 10)
+	var snapshotCount uint64 = 10
+	epc := runEtcdAndCreateSnapshot(t, e2e.CurrentVersion, dataDir, snapshotCount)
+
 	lastVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease)
 	lastVersionStr := lastVersion.String()
-
 	lastClusterVersion := semver.New(lastVersionStr)
 	lastClusterVersion.Patch = 0
 	lastClusterVersionStr := lastClusterVersion.String()
-
 	t.Logf("etcdctl downgrade enable %s", lastVersionStr)
 	downgradeEnable(t, epc, lastVersion)
 
@@ -218,33 +218,41 @@ func TestV2DeprecationSnapshotRecoverOldVersion(t *testing.T) {
 		e2e.AssertProcessLogs(t, epc.Procs[i], "The server is ready to downgrade")
 	}
 
-	cc, err := e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
+	t.Log("Cluster is ready for downgrade")
+
+	t.Log("Adding and removing keys")
+	cc1, err := e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
 	assert.NoError(t, err)
+	addAndRemoveKeysAndMembers(ctx, t, cc1, snapshotCount)
 
-	lastReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
+	cc, err := e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
 	assert.NoError(t, err)
 
-	lastReleaseMemberListResponse, err := cc.MemberList(ctx, false)
+	beforeDowngradeGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
 	assert.NoError(t, err)
 
-	assert.NoError(t, epc.Close())
-	cfg := e2e.ConfigStandalone(*e2e.NewConfig(
-		e2e.WithVersion(e2e.LastVersion),
-		e2e.WithDataDirPath(dataDir),
-	))
-	epc, err = e2e.NewEtcdProcessCluster(context.TODO(), t, e2e.WithConfig(cfg))
+	beforeDowngradeMemberListResponse, err := cc.MemberList(ctx, false)
 	assert.NoError(t, err)
 
+	t.Logf("Starting downgrade process to %q", lastVersionStr)
+	for i := 0; i < len(epc.Procs); i++ {
+		t.Logf("Downgrading member %d by running %s binary", i, e2e.BinPath.EtcdLastRelease)
+		stopEtcd(t, epc.Procs[i])
+		startEtcd(t, epc.Procs[i], e2e.BinPath.EtcdLastRelease)
+	}
+
 	cc, err = e2e.NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC())
 	assert.NoError(t, err)
-	currentReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
+
+	afterDowngradeGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true})
 	assert.NoError(t, err)
 
-	currentReleaseMemberListResponse, err := cc.MemberList(ctx, false)
+	afterDowngradeMemberListResponse, err := cc.MemberList(ctx, false)
 	assert.NoError(t, err)
 
-	assert.Equal(t, lastReleaseGetResponse.Kvs, currentReleaseGetResponse.Kvs)
-	assert.Equal(t, lastReleaseMemberListResponse.Members, currentReleaseMemberListResponse.Members)
+	assert.Equal(t, afterDowngradeGetResponse.Kvs, beforeDowngradeGetResponse.Kvs)
+	assert.Equal(t, afterDowngradeMemberListResponse.Members, beforeDowngradeMemberListResponse.Members)
+
 	assert.NoError(t, epc.Close())
 }
 

From adac92b949a1765bd96b30534876a066b0b9fc52 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 26 May 2023 03:55:18 +0000
Subject: [PATCH 12/44] Add verification for consistent index

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go      | 11 +++++++++++
 server/etcdserver/server_test.go |  4 ++++
 2 files changed, 15 insertions(+)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 76509d088ca..b4b2ffa32d9 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1104,6 +1104,14 @@ func verifySnapshotIndex(snapshot raftpb.Snapshot, cindex uint64) {
 	})
 }
 
+func verifyConsistentIndexIsLatest(snapshot raftpb.Snapshot, cindex uint64) {
+	verify.Verify(func() {
+		if cindex < snapshot.Metadata.Index {
+			panic(fmt.Sprintf("consistent_index(%d) is older than snapshot index (%d)", cindex, snapshot.Metadata.Index))
+		}
+	})
+}
+
 func (s *EtcdServer) applyEntries(ep *etcdProgress, apply *toApply) {
 	if len(apply.entries) == 0 {
 		return
@@ -2083,6 +2091,9 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 			}
 			lg.Panic("failed to create snapshot", zap.Error(err))
 		}
+
+		verifyConsistentIndexIsLatest(snap, s.consistIndex.ConsistentIndex())
+
 		// SaveSnap saves the snapshot to file and appends the corresponding WAL entry.
 		if err = s.r.storage.SaveSnap(snap); err != nil {
 			lg.Panic("failed to save snapshot", zap.Error(err))
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 897f27db416..029527e1df9 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1017,6 +1017,8 @@ func TestSyncTrigger(t *testing.T) {
 
 // TestSnapshot should snapshot the store and cut the persistent
 func TestSnapshot(t *testing.T) {
+	revertFunc := verify.DisableVerifications()
+	defer revertFunc()
 	be, _ := betesting.NewDefaultTmpBackend(t)
 
 	s := raft.NewMemoryStorage()
@@ -1087,6 +1089,8 @@ func TestSnapshot(t *testing.T) {
 
 // TestSnapshotNoV2store should create snapshot using new v2store
 func TestSnapshotNoV2store(t *testing.T) {
+	revertFunc := verify.DisableVerifications()
+	defer revertFunc()
 	be, _ := betesting.NewDefaultTmpBackend(t)
 
 	s := raft.NewMemoryStorage()

From 91359ba2693245e8a270d27cd4940eb117ee1995 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Sat, 27 May 2023 03:30:00 +0000
Subject: [PATCH 13/44] start refactoring recovery logic

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go  | 47 ++++++++++++++++++++++++---------
 server/storage/schema/schema.go | 14 ++++++++++
 2 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index 570b6e63fd9..e1bbb98f4c1 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -379,6 +379,18 @@ func bootstrapClusterWithWAL(cfg config.ServerConfig, meta *snapshotMetadata) (*
 	}, nil
 }
 
+func getRaftSnapshotFromBackend(cfg config.ServerConfig, be backend.Backend) (*raftpb.Snapshot, error) {
+	index, term, confState, err := schema.ReadIndexTermConfState(cfg.Logger, be.ReadTx())
+	if err != nil {
+		cfg.Logger.Panic("failed to recover snapshot info from backend", zap.Error(err))
+	}
+	var rsnap raftpb.Snapshot
+	rsnap.Metadata.Index = index
+	rsnap.Metadata.Term = term
+	rsnap.Metadata.ConfState = *confState
+	return &rsnap, nil
+}
+
 func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) {
 	// Find a snapshot to start/restart a raft node
 	walSnaps, err := wal.ValidSnapshotEntries(cfg.Logger, cfg.WALDir())
@@ -392,21 +404,32 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 		return nil, be, err
 	}
 
-	if snapshot != nil {
-		if err = st.Recovery(snapshot.Data); err != nil {
-			cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
-		}
-
-		if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
-			cfg.Logger.Error("illegal v2store content", zap.Error(err))
+	/*
+		TODO - can snapshot be newer than be with 3.6?
+		snapshot, err := getRaftSnapshotFromBackend(cfg, be)
+		if err != nil {
+			//TODO getRaftSnapshotFromBackend panics for now
 			return nil, be, err
 		}
+	*/
 
-		cfg.Logger.Info(
-			"recovered v2 store from snapshot",
-			zap.Uint64("snapshot-index", snapshot.Metadata.Index),
-			zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
-		)
+	if snapshot != nil {
+		/*
+			if err = st.Recovery(snapshot.Data); err != nil {
+				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
+			}
+
+			if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
+				cfg.Logger.Error("illegal v2store content", zap.Error(err))
+				return nil, be, err
+			}
+
+			cfg.Logger.Info(
+				"recovered v2 store from snapshot",
+				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
+				zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
+			)
+		*/
 
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
diff --git a/server/storage/schema/schema.go b/server/storage/schema/schema.go
index e2e7fe7e566..d96fbe3cd41 100644
--- a/server/storage/schema/schema.go
+++ b/server/storage/schema/schema.go
@@ -21,6 +21,7 @@ import (
 	"go.uber.org/zap"
 
 	"go.etcd.io/etcd/api/v3/version"
+	"go.etcd.io/raft/v3/raftpb"
 
 	"go.etcd.io/etcd/server/v3/storage/backend"
 )
@@ -90,6 +91,19 @@ func DetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, e
 	return UnsafeDetectSchemaVersion(lg, tx)
 }
 
+// ReadIndexTermConfState returns raft index, term and confState from backend.
+func ReadIndexTermConfState(lg *zap.Logger, tx backend.ReadTx) (index uint64, term uint64, confState *raftpb.ConfState, err error) {
+	tx.RLock()
+	defer tx.RUnlock()
+
+	confstate := UnsafeConfStateFromBackend(lg, tx)
+	if confstate == nil {
+		return 0, 0, nil, fmt.Errorf("missing confstate information")
+	}
+	index, term = UnsafeReadConsistentIndex(tx)
+	return index, term, confstate, nil
+}
+
 // UnsafeDetectSchemaVersion non-threadsafe version of DetectSchemaVersion.
 func UnsafeDetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, err error) {
 	vp := UnsafeReadStorageVersion(tx)

From fa9b70b5ebc16e9f259df64b1e4792dfe6bdd56a Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 29 May 2023 03:14:49 +0000
Subject: [PATCH 14/44] update conf change validation

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go   |  8 ++-
 .../etcdserver/api/membership/cluster_test.go | 16 ++++--
 .../api/membership/membership_test.go         | 44 +++++++++++++----
 server/etcdserver/bootstrap.go                | 49 +++++--------------
 server/etcdserver/server.go                   |  3 +-
 server/storage/schema/schema.go               | 14 ------
 6 files changed, 69 insertions(+), 65 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index 5e272cf09ed..5e7f26de2af 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -304,7 +304,8 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 
 // ValidateConfigurationChange takes a proposed ConfChange and
 // ensures that it is still valid.
-func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
+func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange, shouldApplyV3 ShouldApplyV3) error {
+
 	var membersMap map[types.ID]*Member
 	var removedMap map[types.ID]bool
 	if c.v2store != nil {
@@ -312,6 +313,11 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
 	} else {
 		membersMap, removedMap = c.be.MustReadMembersFromBackend()
 	}
+	if !shouldApplyV3 {
+		//TODO geetasg - RaftCluster purely based on backend cannot validate an entry older than ci
+		return nil
+	}
+
 	id := types.ID(cc.NodeID)
 	if removedMap[id] {
 		return ErrIDRemoved
diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go
index 0b33720612e..f455f6c8301 100644
--- a/server/etcdserver/api/membership/cluster_test.go
+++ b/server/etcdserver/api/membership/cluster_test.go
@@ -279,7 +279,10 @@ func TestClusterValidateAndAssignIDs(t *testing.T) {
 
 func TestClusterValidateConfigurationChange(t *testing.T) {
 	cl := NewCluster(zaptest.NewLogger(t), WithMaxLearners(1))
-	cl.SetStore(v2store.New())
+	//cl.SetStore(v2store.New())
+	be := newMembershipBackend()
+	cl.SetBackend(be)
+
 	for i := 1; i <= 4; i++ {
 		var isLearner bool
 		if i == 1 {
@@ -456,7 +459,7 @@ func TestClusterValidateConfigurationChange(t *testing.T) {
 		},
 	}
 	for i, tt := range tests {
-		err := cl.ValidateConfigurationChange(tt.cc)
+		err := cl.ValidateConfigurationChange(tt.cc, ApplyBoth)
 		if err != tt.werr {
 			t.Errorf("#%d: validateConfigurationChange error = %v, want %v", i, err, tt.werr)
 		}
@@ -649,8 +652,10 @@ func TestNodeToMember(t *testing.T) {
 
 func newTestCluster(t testing.TB, membs []*Member) *RaftCluster {
 	c := &RaftCluster{lg: zaptest.NewLogger(t), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
+	be := newMembershipBackend()
+	c.SetBackend(be)
 	for _, m := range membs {
-		c.members[m.ID] = m
+		c.AddMember(m, true)
 	}
 	return c
 }
@@ -996,7 +1001,9 @@ func TestMembershipStore(t *testing.T) {
 	}
 	for i, tt := range tests {
 		c := newTestCluster(t, tt.mems)
-		c.removed = tt.removed
+		for id, _ := range tt.removed {
+			c.RemoveMember(id, true)
+		}
 
 		//snapshot
 		st := v2store.New("/0", "/1")
@@ -1007,6 +1014,7 @@ func TestMembershipStore(t *testing.T) {
 		rst := v2store.New("/0", "/1")
 		rst.Recovery(d)
 		rc := &RaftCluster{lg: zaptest.NewLogger(t), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
+		rc.SetBackend(c.be)
 		rc.SetStore(rst)
 		rc.Recover(func(lg *zap.Logger, v *semver.Version) { return })
 
diff --git a/server/etcdserver/api/membership/membership_test.go b/server/etcdserver/api/membership/membership_test.go
index 728121e1c69..51ce992b3a0 100644
--- a/server/etcdserver/api/membership/membership_test.go
+++ b/server/etcdserver/api/membership/membership_test.go
@@ -22,13 +22,14 @@ import (
 
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/server/v3/etcdserver/version"
+	serverversion "go.etcd.io/etcd/server/v3/etcdserver/version"
 
 	"go.uber.org/zap"
 )
 
 func TestAddRemoveMember(t *testing.T) {
 	c := newTestCluster(t, nil)
-	be := &backendMock{}
+	be := newMembershipBackend()
 	c.SetBackend(be)
 	c.AddMember(newTestMemberAsLearner(17, nil, "node17", nil), true)
 	c.RemoveMember(17, true)
@@ -44,6 +45,7 @@ func TestAddRemoveMember(t *testing.T) {
 		c2 := newTestCluster(t, nil)
 		c2.SetBackend(be)
 		c2.Recover(func(*zap.Logger, *semver.Version) {})
+		//TODO : geetasg - following asserts need to be updated. checkin 63a1cc3fe4 done on 2021-09-30 removes node id 18. Asserts were added on 2021-03-26 via 768da490ed. Confirm and update the asserts - both 17 and 18 are removed.
 		assert.Equal(t, []*Member{{ID: types.ID(18),
 			Attributes: Attributes{Name: "node18"}}}, c2.Members())
 		assert.Equal(t, true, c2.IsIDRemoved(17))
@@ -52,21 +54,45 @@ func TestAddRemoveMember(t *testing.T) {
 }
 
 type backendMock struct {
+	members       map[types.ID]*Member
+	removed       map[types.ID]bool
+	version       *semver.Version
+	downgradeInfo *version.DowngradeInfo
 }
 
 var _ MembershipBackend = (*backendMock)(nil)
 
+func newMembershipBackend() MembershipBackend {
+	return &backendMock{
+		members:       make(map[types.ID]*Member),
+		removed:       make(map[types.ID]bool),
+		downgradeInfo: &serverversion.DowngradeInfo{Enabled: false},
+	}
+}
+
 func (b *backendMock) MustCreateBackendBuckets() {}
 
-func (b *backendMock) ClusterVersionFromBackend() *semver.Version              { return nil }
-func (b *backendMock) MustSaveClusterVersionToBackend(version *semver.Version) {}
+func (b *backendMock) ClusterVersionFromBackend() *semver.Version { return b.version }
+func (b *backendMock) MustSaveClusterVersionToBackend(version *semver.Version) {
+	b.version = version
+}
 
 func (b *backendMock) MustReadMembersFromBackend() (x map[types.ID]*Member, y map[types.ID]bool) {
-	return
+	return b.members, b.removed
+}
+func (b *backendMock) MustSaveMemberToBackend(m *Member) {
+	b.members[m.ID] = m
+}
+func (b *backendMock) TrimMembershipFromBackend() error {
+	b.members = make(map[types.ID]*Member)
+	b.removed = make(map[types.ID]bool)
+	return nil
+}
+func (b *backendMock) MustDeleteMemberFromBackend(id types.ID) {
+	b.removed[id] = true
 }
-func (b *backendMock) MustSaveMemberToBackend(*Member)      {}
-func (b *backendMock) TrimMembershipFromBackend() error     { return nil }
-func (b *backendMock) MustDeleteMemberFromBackend(types.ID) {}
 
-func (b *backendMock) MustSaveDowngradeToBackend(*version.DowngradeInfo) {}
-func (b *backendMock) DowngradeInfoFromBackend() *version.DowngradeInfo  { return nil }
+func (b *backendMock) MustSaveDowngradeToBackend(downgradeInfo *version.DowngradeInfo) {
+	b.downgradeInfo = downgradeInfo
+}
+func (b *backendMock) DowngradeInfoFromBackend() *version.DowngradeInfo { return b.downgradeInfo }
diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index e1bbb98f4c1..93f65f29637 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -379,18 +379,6 @@ func bootstrapClusterWithWAL(cfg config.ServerConfig, meta *snapshotMetadata) (*
 	}, nil
 }
 
-func getRaftSnapshotFromBackend(cfg config.ServerConfig, be backend.Backend) (*raftpb.Snapshot, error) {
-	index, term, confState, err := schema.ReadIndexTermConfState(cfg.Logger, be.ReadTx())
-	if err != nil {
-		cfg.Logger.Panic("failed to recover snapshot info from backend", zap.Error(err))
-	}
-	var rsnap raftpb.Snapshot
-	rsnap.Metadata.Index = index
-	rsnap.Metadata.Term = term
-	rsnap.Metadata.ConfState = *confState
-	return &rsnap, nil
-}
-
 func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) {
 	// Find a snapshot to start/restart a raft node
 	walSnaps, err := wal.ValidSnapshotEntries(cfg.Logger, cfg.WALDir())
@@ -404,32 +392,21 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 		return nil, be, err
 	}
 
-	/*
-		TODO - can snapshot be newer than be with 3.6?
-		snapshot, err := getRaftSnapshotFromBackend(cfg, be)
-		if err != nil {
-			//TODO getRaftSnapshotFromBackend panics for now
-			return nil, be, err
-		}
-	*/
-
 	if snapshot != nil {
-		/*
-			if err = st.Recovery(snapshot.Data); err != nil {
-				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
-			}
+		if err = st.Recovery(snapshot.Data); err != nil {
+			cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
+		}
 
-			if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
-				cfg.Logger.Error("illegal v2store content", zap.Error(err))
-				return nil, be, err
-			}
+		if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
+			cfg.Logger.Error("illegal v2store content", zap.Error(err))
+			return nil, be, err
+		}
 
-			cfg.Logger.Info(
-				"recovered v2 store from snapshot",
-				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
-				zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
-			)
-		*/
+		cfg.Logger.Info(
+			"recovered v2 store from snapshot",
+			zap.Uint64("snapshot-index", snapshot.Metadata.Index),
+			zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
+		)
 
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
@@ -470,7 +447,7 @@ func (c *bootstrapedCluster) Finalize(cfg config.ServerConfig, s *bootstrappedSt
 	if !s.wal.haveWAL {
 		c.cl.SetID(c.nodeID, c.cl.ID())
 	}
-	c.cl.SetStore(s.st)
+	//c.cl.SetStore(s.st)
 	c.cl.SetBackend(schema.NewMembershipBackend(cfg.Logger, s.backend.be))
 	if s.wal.haveWAL {
 		c.cl.Recover(api.UpdateCapability)
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index b4b2ffa32d9..e18e8cdfebc 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1982,7 +1982,8 @@ func removeNeedlessRangeReqs(txn *pb.TxnRequest) {
 // applyConfChange applies a ConfChange to the server. It is only
 // invoked with a ConfChange that has already passed through Raft
 func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.ConfState, shouldApplyV3 membership.ShouldApplyV3) (bool, error) {
-	if err := s.cluster.ValidateConfigurationChange(cc); err != nil {
+	//while recovering WAL from scratch - store starts empty and returns ValidateConfigurationChange err nil. backend on the other hand starts stateful and returns "member already exists" etc errors on this Validation and enters the loop below
+	if err := s.cluster.ValidateConfigurationChange(cc, shouldApplyV3); err != nil && membership.ApplyBoth == shouldApplyV3 {
 		cc.NodeID = raft.None
 		s.r.ApplyConfChange(cc)
 
diff --git a/server/storage/schema/schema.go b/server/storage/schema/schema.go
index d96fbe3cd41..e2e7fe7e566 100644
--- a/server/storage/schema/schema.go
+++ b/server/storage/schema/schema.go
@@ -21,7 +21,6 @@ import (
 	"go.uber.org/zap"
 
 	"go.etcd.io/etcd/api/v3/version"
-	"go.etcd.io/raft/v3/raftpb"
 
 	"go.etcd.io/etcd/server/v3/storage/backend"
 )
@@ -91,19 +90,6 @@ func DetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, e
 	return UnsafeDetectSchemaVersion(lg, tx)
 }
 
-// ReadIndexTermConfState returns raft index, term and confState from backend.
-func ReadIndexTermConfState(lg *zap.Logger, tx backend.ReadTx) (index uint64, term uint64, confState *raftpb.ConfState, err error) {
-	tx.RLock()
-	defer tx.RUnlock()
-
-	confstate := UnsafeConfStateFromBackend(lg, tx)
-	if confstate == nil {
-		return 0, 0, nil, fmt.Errorf("missing confstate information")
-	}
-	index, term = UnsafeReadConsistentIndex(tx)
-	return index, term, confstate, nil
-}
-
 // UnsafeDetectSchemaVersion non-threadsafe version of DetectSchemaVersion.
 func UnsafeDetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, err error) {
 	vp := UnsafeReadStorageVersion(tx)

From 543d3055a6a77b26626320eb7b87aab9d0dfefa9 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 29 May 2023 20:28:26 +0000
Subject: [PATCH 15/44] Stop v2 recovery during applySnapshot and bootstrap

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go | 14 --------------
 server/etcdserver/server.go    | 11 -----------
 2 files changed, 25 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index 93f65f29637..166d0f478fe 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -393,20 +393,6 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 	}
 
 	if snapshot != nil {
-		if err = st.Recovery(snapshot.Data); err != nil {
-			cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
-		}
-
-		if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
-			cfg.Logger.Error("illegal v2store content", zap.Error(err))
-			return nil, be, err
-		}
-
-		cfg.Logger.Info(
-			"recovered v2 store from snapshot",
-			zap.Uint64("snapshot-index", snapshot.Metadata.Index),
-			zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
-		)
 
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index e18e8cdfebc..67a111ceca7 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1046,17 +1046,6 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) {
 		lg.Info("restored auth store")
 	}
 
-	lg.Info("restoring v2 store")
-	if err := s.v2store.Recovery(toApply.snapshot.Data); err != nil {
-		lg.Panic("failed to restore v2 store", zap.Error(err))
-	}
-
-	if err := serverstorage.AssertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil {
-		lg.Panic("illegal v2store content", zap.Error(err))
-	}
-
-	lg.Info("restored v2 store")
-
 	s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe))
 
 	lg.Info("restoring cluster configuration")

From 8321280f2a200facd766fd1d4e48e9dabc2f36b4 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 29 May 2023 21:13:48 +0000
Subject: [PATCH 16/44] Deactivate v2store recovery for non-test only

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/config/v2_deprecation.go |  5 ++---
 server/etcdserver/bootstrap.go  | 16 ++++++++++++++++
 server/etcdserver/server.go     | 13 +++++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/server/config/v2_deprecation.go b/server/config/v2_deprecation.go
index 862c3bb9343..c1bf8e41bf7 100644
--- a/server/config/v2_deprecation.go
+++ b/server/config/v2_deprecation.go
@@ -24,11 +24,10 @@ const (
 	V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only")
 	// V2store is WIPED if found !!!
 	V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data")
-	// V2store is neither written nor read. Usage of this configuration is blocking
-	// ability to rollback to etcd v3.5.
+	// V2store is neither written nor read for 3.6. v2snapshot is published only for backward compatibility
 	V2_DEPR_2_GONE = V2DeprecationEnum("gone")
 
-	V2_DEPR_DEFAULT = V2_DEPR_1_WRITE_ONLY
+	V2_DEPR_DEFAULT = V2_DEPR_2_GONE
 )
 
 func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool {
diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index 166d0f478fe..d72843433af 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -393,6 +393,22 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 	}
 
 	if snapshot != nil {
+		if cfg.V2Deprecation != config.V2_DEPR_2_GONE {
+			if err = st.Recovery(snapshot.Data); err != nil {
+				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
+			}
+
+			if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
+				cfg.Logger.Error("illegal v2store content", zap.Error(err))
+				return nil, be, err
+			}
+
+			cfg.Logger.Info(
+				"recovered v2 store from snapshot",
+				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
+				zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
+			)
+		}
 
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 67a111ceca7..a0da1957769 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1046,6 +1046,19 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) {
 		lg.Info("restored auth store")
 	}
 
+	if s.Cfg.V2Deprecation != config.V2_DEPR_2_GONE {
+		lg.Info("restoring v2 store")
+		if err := s.v2store.Recovery(toApply.snapshot.Data); err != nil {
+			lg.Panic("failed to restore v2 store", zap.Error(err))
+		}
+
+		if err := serverstorage.AssertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil {
+			lg.Panic("illegal v2store content", zap.Error(err))
+		}
+
+		lg.Info("restored v2 store")
+	}
+
 	s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe))
 
 	lg.Info("restoring cluster configuration")

From b44cb381461d316fd90bc07d266a42f236fad969 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 30 May 2023 18:00:33 +0000
Subject: [PATCH 17/44] Adding comment to check back on v2discovery support

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index d72843433af..e42144e69df 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -325,6 +325,7 @@ func bootstrapNewClusterNoWAL(cfg config.ServerConfig, prt http.RoundTripper) (*
 	}
 	if cfg.ShouldDiscover() {
 		var str string
+		//TODO geetasg check about v2 discovery support
 		if cfg.DiscoveryURL != "" {
 			cfg.Logger.Warn("V2 discovery is deprecated!")
 			str, err = v2discovery.JoinCluster(cfg.Logger, cfg.DiscoveryURL, cfg.DiscoveryProxy, m.ID, cfg.InitialPeerURLsMap.String())

From d8d0f622b6bd74f58c7f8f3a58689a1d20f53734 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 30 May 2023 22:24:23 +0000
Subject: [PATCH 18/44] Remove SetStore from etcdutl snapshot restore path

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 etcdutl/snapshot/v3_snapshot.go                  | 6 +++---
 server/etcdserver/api/membership/cluster_test.go | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/etcdutl/snapshot/v3_snapshot.go b/etcdutl/snapshot/v3_snapshot.go
index 8958ba80da1..ff19277f8b6 100644
--- a/etcdutl/snapshot/v3_snapshot.go
+++ b/etcdutl/snapshot/v3_snapshot.go
@@ -396,8 +396,6 @@ func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) {
 	}
 
 	// add members again to persist them to the store we create.
-	st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
-	s.cl.SetStore(st)
 	be := backend.NewDefaultBackend(s.lg, s.outDbPath())
 	defer be.Close()
 	s.cl.SetBackend(schema.NewMembershipBackend(s.lg, be))
@@ -457,7 +455,9 @@ func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) {
 		return nil, err
 	}
 
-	b, berr := st.Save()
+	st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
+	s.cl.Store(st)
+	b, berr := st.SaveNoCopy()
 	if berr != nil {
 		return nil, berr
 	}
diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go
index f455f6c8301..acf2d645d17 100644
--- a/server/etcdserver/api/membership/cluster_test.go
+++ b/server/etcdserver/api/membership/cluster_test.go
@@ -279,7 +279,6 @@ func TestClusterValidateAndAssignIDs(t *testing.T) {
 
 func TestClusterValidateConfigurationChange(t *testing.T) {
 	cl := NewCluster(zaptest.NewLogger(t), WithMaxLearners(1))
-	//cl.SetStore(v2store.New())
 	be := newMembershipBackend()
 	cl.SetBackend(be)
 

From dc2744fe5ab032d2e8ad8bd6bf3745c54b5644e8 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 30 May 2023 22:59:18 +0000
Subject: [PATCH 19/44] Update RaftCluster tests

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster_test.go | 9 ++++-----
 server/etcdserver/bootstrap.go                   | 1 -
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go
index acf2d645d17..566eed530fa 100644
--- a/server/etcdserver/api/membership/cluster_test.go
+++ b/server/etcdserver/api/membership/cluster_test.go
@@ -477,7 +477,6 @@ func TestClusterGenID(t *testing.T) {
 	}
 	previd := cs.ID()
 
-	cs.SetStore(mockstore.NewNop())
 	cs.AddMember(newTestMember(3, nil, "", nil), true)
 	cs.genID()
 	if cs.ID() == previd {
@@ -520,8 +519,8 @@ func TestNodeToMemberBad(t *testing.T) {
 func TestClusterAddMember(t *testing.T) {
 	st := mockstore.NewRecorder()
 	c := newTestCluster(t, nil)
-	c.SetStore(st)
 	c.AddMember(newTestMember(1, nil, "node1", nil), true)
+	c.Store(st)
 
 	wactions := []testutil.Action{
 		{
@@ -543,8 +542,8 @@ func TestClusterAddMember(t *testing.T) {
 func TestClusterAddMemberAsLearner(t *testing.T) {
 	st := mockstore.NewRecorder()
 	c := newTestCluster(t, nil)
-	c.SetStore(st)
 	c.AddMember(newTestMemberAsLearner(1, nil, "node1", nil), true)
+	c.Store(st)
 
 	wactions := []testutil.Action{
 		{
@@ -586,8 +585,8 @@ func TestClusterMembers(t *testing.T) {
 func TestClusterRemoveMember(t *testing.T) {
 	st := mockstore.NewRecorder()
 	c := newTestCluster(t, nil)
-	c.SetStore(st)
 	c.RemoveMember(1, true)
+	c.Store(st)
 
 	wactions := []testutil.Action{
 		{Name: "Delete", Params: []interface{}{MemberStoreKey(1), true, true}},
@@ -1009,7 +1008,7 @@ func TestMembershipStore(t *testing.T) {
 		c.Store(st)
 		d, _ := st.SaveNoCopy()
 
-		//recover from snapshot
+		//3.5 recover from snapshot
 		rst := v2store.New("/0", "/1")
 		rst.Recovery(d)
 		rc := &RaftCluster{lg: zaptest.NewLogger(t), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index e42144e69df..de5828c9439 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -450,7 +450,6 @@ func (c *bootstrapedCluster) Finalize(cfg config.ServerConfig, s *bootstrappedSt
 	if !s.wal.haveWAL {
 		c.cl.SetID(c.nodeID, c.cl.ID())
 	}
-	//c.cl.SetStore(s.st)
 	c.cl.SetBackend(schema.NewMembershipBackend(cfg.Logger, s.backend.be))
 	if s.wal.haveWAL {
 		c.cl.Recover(api.UpdateCapability)

From 0ce660340eb38ce9bbc6c2ae3bebe9c9bd690e6a Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 30 May 2023 23:50:11 +0000
Subject: [PATCH 20/44] Refactor to create a function to return membership in
 v2 format

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 etcdutl/snapshot/v3_snapshot.go   |  9 +--------
 server/etcdserver/cluster_util.go | 12 ++++++++++++
 server/etcdserver/server.go       | 11 +----------
 3 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/etcdutl/snapshot/v3_snapshot.go b/etcdutl/snapshot/v3_snapshot.go
index ff19277f8b6..03ee00d6285 100644
--- a/etcdutl/snapshot/v3_snapshot.go
+++ b/etcdutl/snapshot/v3_snapshot.go
@@ -38,7 +38,6 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/schema"
@@ -455,17 +454,11 @@ func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) {
 		return nil, err
 	}
 
-	st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
-	s.cl.Store(st)
-	b, berr := st.SaveNoCopy()
-	if berr != nil {
-		return nil, berr
-	}
 	confState := raftpb.ConfState{
 		Voters: nodeIDs,
 	}
 	raftSnap := raftpb.Snapshot{
-		Data: b,
+		Data: etcdserver.GetMembershipInfoInV2Format(s.lg, s.cl),
 		Metadata: raftpb.SnapshotMetadata{
 			Index:     commit,
 			Term:      term,
diff --git a/server/etcdserver/cluster_util.go b/server/etcdserver/cluster_util.go
index 065283a5855..220045c0590 100644
--- a/server/etcdserver/cluster_util.go
+++ b/server/etcdserver/cluster_util.go
@@ -28,6 +28,7 @@ import (
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/errors"
 
 	"github.com/coreos/go-semver/semver"
@@ -416,3 +417,14 @@ func convertToClusterVersion(v string) (*semver.Version, error) {
 	ver = &semver.Version{Major: ver.Major, Minor: ver.Minor}
 	return ver, nil
 }
+
+func GetMembershipInfoInV2Format(lg *zap.Logger, cl *membership.RaftCluster) []byte {
+	var st v2store.Store
+	st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
+	cl.Store(st)
+	d, err := st.SaveNoCopy()
+	if err != nil {
+		lg.Panic("failed to save v2 store", zap.Error(err))
+	}
+	return d
+}
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index a0da1957769..1b3fa49d6e0 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -2076,16 +2076,7 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 
 	s.GoAttach(func() {
 		lg := s.Logger()
-		var st v2store.Store
-		st = v2store.New(StoreClusterPrefix, StoreKeysPrefix)
-		s.cluster.Store(st)
-		d, err := st.SaveNoCopy()
-		// TODO: current store will never fail to do a snapshot
-		// what should we do if the store might fail?
-		if err != nil {
-			lg.Panic("failed to save v2 store", zap.Error(err))
-		}
-		snap, err := s.r.raftStorage.CreateSnapshot(snapi, &confState, d)
+		snap, err := s.r.raftStorage.CreateSnapshot(snapi, &confState, GetMembershipInfoInV2Format(lg, s.cluster))
 		if err != nil {
 			// the snapshot was done asynchronously with the progress of raft.
 			// raft might have already got a newer snapshot.

From 19762670f55f129879751e4477b8aeba3e11a5be Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 00:28:09 +0000
Subject: [PATCH 21/44] Update backup command logic for saving membership to
 v2store

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 etcdutl/etcdutl/backup_command.go | 37 +++++++------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/etcdutl/etcdutl/backup_command.go b/etcdutl/etcdutl/backup_command.go
index 89121a37e95..3a4ec58ad2c 100644
--- a/etcdutl/etcdutl/backup_command.go
+++ b/etcdutl/etcdutl/backup_command.go
@@ -28,9 +28,9 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
+	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
 	"go.etcd.io/etcd/server/v3/storage/schema"
@@ -131,10 +131,12 @@ func HandleBackup(withV3 bool, srcDir string, destDir string, srcWAL string, des
 	destDbPath := datadir.ToBackendFileName(destDir)
 	srcDbPath := datadir.ToBackendFileName(srcDir)
 	desired := newDesiredCluster()
+	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
+	raftCluster.SetID(desired.nodeId, desired.clusterId)
 
-	walsnap := saveSnap(lg, destSnap, srcSnap, &desired)
+	walsnap := saveSnap(lg, destSnap, srcSnap, &desired, raftCluster)
 	metadata, state, ents := translateWAL(lg, srcWAL, walsnap)
-	saveDB(lg, destDbPath, srcDbPath, state.Commit, state.Term, &desired)
+	saveDB(lg, destDbPath, srcDbPath, state.Commit, state.Term, &desired, raftCluster)
 
 	neww, err := wal.Create(lg, destWAL, pbutil.MustMarshal(&metadata))
 	if err != nil {
@@ -157,7 +159,7 @@ func HandleBackup(withV3 bool, srcDir string, destDir string, srcWAL string, des
 	return nil
 }
 
-func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster) (walsnap walpb.Snapshot) {
+func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster, raftCluster *membership.RaftCluster) (walsnap walpb.Snapshot) {
 	ss := snap.New(lg, srcSnap)
 	snapshot, err := ss.Load()
 	if err != nil && err != snap.ErrNoSnapshot {
@@ -167,7 +169,7 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster)
 		walsnap.Index, walsnap.Term, walsnap.ConfState = snapshot.Metadata.Index, snapshot.Metadata.Term, &desired.confState
 		newss := snap.New(lg, destSnap)
 		snapshot.Metadata.ConfState = desired.confState
-		snapshot.Data = mustTranslateV2store(lg, snapshot.Data, desired)
+		snapshot.Data = etcdserver.GetMembershipInfoInV2Format(lg, raftCluster)
 		if err = newss.SaveSnap(*snapshot); err != nil {
 			lg.Fatal("saveSnap(Snapshoter.SaveSnap) failed", zap.Error(err))
 		}
@@ -175,26 +177,6 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster)
 	return walsnap
 }
 
-// mustTranslateV2store processes storeData such that they match 'desiredCluster'.
-// In particular the method overrides membership information.
-func mustTranslateV2store(lg *zap.Logger, storeData []byte, desired *desiredCluster) []byte {
-	st := v2store.New()
-	if err := st.Recovery(storeData); err != nil {
-		lg.Panic("cannot translate v2store", zap.Error(err))
-	}
-
-	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
-	raftCluster.SetID(desired.nodeId, desired.clusterId)
-	raftCluster.SetStore(st)
-	raftCluster.PushMembershipToStorage()
-
-	outputData, err := st.Save()
-	if err != nil {
-		lg.Panic("cannot save v2store", zap.Error(err))
-	}
-	return outputData
-}
-
 func translateWAL(lg *zap.Logger, srcWAL string, walsnap walpb.Snapshot) (etcdserverpb.Metadata, raftpb.HardState, []raftpb.Entry) {
 	w, err := wal.OpenForRead(lg, srcWAL, walsnap)
 	if err != nil {
@@ -266,7 +248,7 @@ func raftEntryToNoOp(entry *raftpb.Entry) {
 }
 
 // saveDB copies the v3 backend and strips cluster information.
-func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desired *desiredCluster) {
+func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desired *desiredCluster, raftCluster *membership.RaftCluster) {
 	// open src db to safely copy db state
 	var src *bolt.DB
 	ch := make(chan *bolt.DB, 1)
@@ -309,9 +291,6 @@ func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desir
 	if err := ms.TrimClusterFromBackend(); err != nil {
 		lg.Fatal("bbolt tx.Membership failed", zap.Error(err))
 	}
-
-	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
-	raftCluster.SetID(desired.nodeId, desired.clusterId)
 	raftCluster.SetBackend(ms)
 	raftCluster.PushMembershipToStorage()
 }

From 0b4c9c65e65f1386413a9bc08d143ef209a018f2 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 02:28:07 +0000
Subject: [PATCH 22/44] Revert "Update backup command logic for saving
 membership to v2store" This reverts commit
 19762670f55f129879751e4477b8aeba3e11a5be. Robustness test failed..

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 etcdutl/etcdutl/backup_command.go | 37 ++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/etcdutl/etcdutl/backup_command.go b/etcdutl/etcdutl/backup_command.go
index 3a4ec58ad2c..89121a37e95 100644
--- a/etcdutl/etcdutl/backup_command.go
+++ b/etcdutl/etcdutl/backup_command.go
@@ -28,9 +28,9 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
-	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
 	"go.etcd.io/etcd/server/v3/storage/schema"
@@ -131,12 +131,10 @@ func HandleBackup(withV3 bool, srcDir string, destDir string, srcWAL string, des
 	destDbPath := datadir.ToBackendFileName(destDir)
 	srcDbPath := datadir.ToBackendFileName(srcDir)
 	desired := newDesiredCluster()
-	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
-	raftCluster.SetID(desired.nodeId, desired.clusterId)
 
-	walsnap := saveSnap(lg, destSnap, srcSnap, &desired, raftCluster)
+	walsnap := saveSnap(lg, destSnap, srcSnap, &desired)
 	metadata, state, ents := translateWAL(lg, srcWAL, walsnap)
-	saveDB(lg, destDbPath, srcDbPath, state.Commit, state.Term, &desired, raftCluster)
+	saveDB(lg, destDbPath, srcDbPath, state.Commit, state.Term, &desired)
 
 	neww, err := wal.Create(lg, destWAL, pbutil.MustMarshal(&metadata))
 	if err != nil {
@@ -159,7 +157,7 @@ func HandleBackup(withV3 bool, srcDir string, destDir string, srcWAL string, des
 	return nil
 }
 
-func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster, raftCluster *membership.RaftCluster) (walsnap walpb.Snapshot) {
+func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster) (walsnap walpb.Snapshot) {
 	ss := snap.New(lg, srcSnap)
 	snapshot, err := ss.Load()
 	if err != nil && err != snap.ErrNoSnapshot {
@@ -169,7 +167,7 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster,
 		walsnap.Index, walsnap.Term, walsnap.ConfState = snapshot.Metadata.Index, snapshot.Metadata.Term, &desired.confState
 		newss := snap.New(lg, destSnap)
 		snapshot.Metadata.ConfState = desired.confState
-		snapshot.Data = etcdserver.GetMembershipInfoInV2Format(lg, raftCluster)
+		snapshot.Data = mustTranslateV2store(lg, snapshot.Data, desired)
 		if err = newss.SaveSnap(*snapshot); err != nil {
 			lg.Fatal("saveSnap(Snapshoter.SaveSnap) failed", zap.Error(err))
 		}
@@ -177,6 +175,26 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster,
 	return walsnap
 }
 
+// mustTranslateV2store processes storeData such that they match 'desiredCluster'.
+// In particular the method overrides membership information.
+func mustTranslateV2store(lg *zap.Logger, storeData []byte, desired *desiredCluster) []byte {
+	st := v2store.New()
+	if err := st.Recovery(storeData); err != nil {
+		lg.Panic("cannot translate v2store", zap.Error(err))
+	}
+
+	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
+	raftCluster.SetID(desired.nodeId, desired.clusterId)
+	raftCluster.SetStore(st)
+	raftCluster.PushMembershipToStorage()
+
+	outputData, err := st.Save()
+	if err != nil {
+		lg.Panic("cannot save v2store", zap.Error(err))
+	}
+	return outputData
+}
+
 func translateWAL(lg *zap.Logger, srcWAL string, walsnap walpb.Snapshot) (etcdserverpb.Metadata, raftpb.HardState, []raftpb.Entry) {
 	w, err := wal.OpenForRead(lg, srcWAL, walsnap)
 	if err != nil {
@@ -248,7 +266,7 @@ func raftEntryToNoOp(entry *raftpb.Entry) {
 }
 
 // saveDB copies the v3 backend and strips cluster information.
-func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desired *desiredCluster, raftCluster *membership.RaftCluster) {
+func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desired *desiredCluster) {
 	// open src db to safely copy db state
 	var src *bolt.DB
 	ch := make(chan *bolt.DB, 1)
@@ -291,6 +309,9 @@ func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desir
 	if err := ms.TrimClusterFromBackend(); err != nil {
 		lg.Fatal("bbolt tx.Membership failed", zap.Error(err))
 	}
+
+	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
+	raftCluster.SetID(desired.nodeId, desired.clusterId)
 	raftCluster.SetBackend(ms)
 	raftCluster.PushMembershipToStorage()
 }

From 83735478f156f682867a0b769970b8403a840546 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 02:49:05 +0000
Subject: [PATCH 23/44] Update server tests

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go    | 15 ++++++---------
 server/etcdserver/snapshot_merge.go |  6 +-----
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 029527e1df9..902ea55e92e 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -648,13 +648,13 @@ func TestApplyConfChangeShouldStop(t *testing.T) {
 // where consistIndex equals to applied index.
 func TestApplyConfigChangeUpdatesConsistIndex(t *testing.T) {
 	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
 
 	cl := membership.NewCluster(zaptest.NewLogger(t))
-	cl.SetStore(v2store.New())
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: types.ID(1)}, true)
 
-	be, _ := betesting.NewDefaultTmpBackend(t)
-	defer betesting.Close(t, be)
 	schema.CreateMetaBucket(be.BatchTx())
 
 	ci := cindex.NewConsistentIndex(be)
@@ -1148,9 +1148,9 @@ func TestSnapshotOrdering(t *testing.T) {
 
 	lg := zaptest.NewLogger(t)
 	n := newNopReadyNode()
-	st := v2store.New()
 	cl := membership.NewCluster(lg)
-	cl.SetStore(st)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 
 	testdir := t.TempDir()
 
@@ -1170,21 +1170,18 @@ func TestSnapshotOrdering(t *testing.T) {
 		storage:     p,
 		raftStorage: rs,
 	})
-	be, _ := betesting.NewDefaultTmpBackend(t)
 	ci := cindex.NewConsistentIndex(be)
 	s := &EtcdServer{
 		lgMu:         new(sync.RWMutex),
 		lg:           lg,
-		Cfg:          config.ServerConfig{Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries},
+		Cfg:          config.ServerConfig{Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, V2Deprecation: config.V2_DEPR_2_GONE},
 		r:            *r,
-		v2store:      st,
 		snapshotter:  snap.New(lg, snapdir),
 		cluster:      cl,
 		SyncTicker:   &time.Ticker{},
 		consistIndex: ci,
 		beHooks:      serverstorage.NewBackendHooks(lg, ci),
 	}
-	s.applyV2 = &applierV2store{store: s.v2store, cluster: s.cluster}
 
 	s.kv = mvcc.New(lg, be, &lease.FakeLessor{}, mvcc.StoreConfig{})
 	s.be = be
diff --git a/server/etcdserver/snapshot_merge.go b/server/etcdserver/snapshot_merge.go
index 963ead5a7e2..5afbc626c18 100644
--- a/server/etcdserver/snapshot_merge.go
+++ b/server/etcdserver/snapshot_merge.go
@@ -31,11 +31,7 @@ import (
 func (s *EtcdServer) createMergedSnapshotMessage(m raftpb.Message, snapt, snapi uint64, confState raftpb.ConfState) snap.Message {
 	lg := s.Logger()
 	// get a snapshot of v2 store as []byte
-	clone := s.v2store.Clone()
-	d, err := clone.SaveNoCopy()
-	if err != nil {
-		lg.Panic("failed to save v2 store data", zap.Error(err))
-	}
+	d := GetMembershipInfoInV2Format(lg, s.cluster)
 
 	// commit kv to write metadata(for example: consistent index).
 	s.KV().Commit()

From ea7d7f0dd3cebeee0a1969c22e7a52cd95869ea7 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 04:36:42 +0000
Subject: [PATCH 24/44] update server unit tests

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 47 ++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 902ea55e92e..5f4dc27791c 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -513,8 +513,13 @@ func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
 }
 
 func TestApplyConfChangeError(t *testing.T) {
-	cl := membership.NewCluster(zaptest.NewLogger(t))
-	cl.SetStore(v2store.New())
+	lg := zaptest.NewLogger(t)
+
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+	cl := membership.NewCluster(lg)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
+
 	for i := 1; i <= 4; i++ {
 		cl.AddMember(&membership.Member{ID: types.ID(i)}, true)
 	}
@@ -579,7 +584,7 @@ func TestApplyConfChangeError(t *testing.T) {
 		n := newNodeRecorder()
 		srv := &EtcdServer{
 			lgMu:    new(sync.RWMutex),
-			lg:      zaptest.NewLogger(t),
+			lg:      lg,
 			r:       *newRaftNode(raftNodeConfig{lg: zaptest.NewLogger(t), Node: n}),
 			cluster: cl,
 		}
@@ -601,17 +606,22 @@ func TestApplyConfChangeError(t *testing.T) {
 }
 
 func TestApplyConfChangeShouldStop(t *testing.T) {
-	cl := membership.NewCluster(zaptest.NewLogger(t))
-	cl.SetStore(v2store.New())
+	lg := zaptest.NewLogger(t)
+
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+	cl := membership.NewCluster(lg)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
+
 	for i := 1; i <= 3; i++ {
 		cl.AddMember(&membership.Member{ID: types.ID(i)}, true)
 	}
 	r := newRaftNode(raftNodeConfig{
-		lg:        zaptest.NewLogger(t),
+		lg:        lg,
 		Node:      newNodeNop(),
 		transport: newNopTransporter(),
 	})
-	lg := zaptest.NewLogger(t)
+
 	srv := &EtcdServer{
 		lgMu:     new(sync.RWMutex),
 		lg:       lg,
@@ -651,7 +661,7 @@ func TestApplyConfigChangeUpdatesConsistIndex(t *testing.T) {
 	be, _ := betesting.NewDefaultTmpBackend(t)
 	defer betesting.Close(t, be)
 
-	cl := membership.NewCluster(zaptest.NewLogger(t))
+	cl := membership.NewCluster(lg)
 	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: types.ID(1)}, true)
 
@@ -728,8 +738,12 @@ func realisticRaftNode(lg *zap.Logger) *raftNode {
 // if the local member is removed along with other conf updates.
 func TestApplyMultiConfChangeShouldStop(t *testing.T) {
 	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+
 	cl := membership.NewCluster(lg)
-	cl.SetStore(v2store.New())
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
+
 	for i := 1; i <= 5; i++ {
 		cl.AddMember(&membership.Member{ID: types.ID(i)}, true)
 	}
@@ -1403,8 +1417,8 @@ func TestAddMember(t *testing.T) {
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
 	cl := newTestCluster(t, nil)
-	st := v2store.New()
-	cl.SetStore(st)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	r := newRaftNode(raftNodeConfig{
 		lg:          lg,
 		Node:        n,
@@ -1416,7 +1430,6 @@ func TestAddMember(t *testing.T) {
 		lgMu:         new(sync.RWMutex),
 		lg:           lg,
 		r:            *r,
-		v2store:      st,
 		cluster:      cl,
 		reqIDGen:     idutil.NewGenerator(0, time.Time{}),
 		SyncTicker:   &time.Ticker{},
@@ -1449,8 +1462,8 @@ func TestRemoveMember(t *testing.T) {
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
 	cl := newTestCluster(t, nil)
-	st := v2store.New()
-	cl.SetStore(v2store.New())
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: 1234}, true)
 	r := newRaftNode(raftNodeConfig{
 		lg:          lg,
@@ -1463,7 +1476,6 @@ func TestRemoveMember(t *testing.T) {
 		lgMu:         new(sync.RWMutex),
 		lg:           zaptest.NewLogger(t),
 		r:            *r,
-		v2store:      st,
 		cluster:      cl,
 		reqIDGen:     idutil.NewGenerator(0, time.Time{}),
 		SyncTicker:   &time.Ticker{},
@@ -1495,8 +1507,8 @@ func TestUpdateMember(t *testing.T) {
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
 	cl := newTestCluster(t, nil)
-	st := v2store.New()
-	cl.SetStore(st)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: 1234}, true)
 	r := newRaftNode(raftNodeConfig{
 		lg:          lg,
@@ -1509,7 +1521,6 @@ func TestUpdateMember(t *testing.T) {
 		lgMu:         new(sync.RWMutex),
 		lg:           lg,
 		r:            *r,
-		v2store:      st,
 		cluster:      cl,
 		reqIDGen:     idutil.NewGenerator(0, time.Time{}),
 		SyncTicker:   &time.Ticker{},

From 7dce3103b775510fa6eeff802172ec1412cd3558 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 19:31:02 +0000
Subject: [PATCH 25/44] Update consumers of RaftCluster SetStore

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go      |  5 +++++
 server/etcdserver/api/membership/cluster_test.go |  7 ++++---
 server/etcdserver/server_test.go                 | 11 +++++++++--
 tests/e2e/v2store_deprecation_test.go            |  7 ++-----
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index 91661ad1574..f7fe1842690 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -888,3 +888,8 @@ func (c *RaftCluster) Store(store v2store.Store) {
 		mustSaveClusterVersionToStore(c.lg, store, c.version)
 	}
 }
+
+func (c *RaftCluster) RecoverMembersFromStore(st v2store.Store) {
+	c.members, c.removed = membersFromStore(c.lg, st)
+	c.buildMembershipMetric()
+}
diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go
index 566eed530fa..64b43934b3f 100644
--- a/server/etcdserver/api/membership/cluster_test.go
+++ b/server/etcdserver/api/membership/cluster_test.go
@@ -21,8 +21,6 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/coreos/go-semver/semver"
-	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
@@ -981,6 +979,8 @@ func TestIsReadyToPromoteMember(t *testing.T) {
 	}
 }
 
+/*
+covered by TestV2DeprecationSnapshotMatches
 // TestMembershipStore tests code path used by snapshot
 func TestMembershipStore(t *testing.T) {
 	name := "etcd"
@@ -1013,7 +1013,7 @@ func TestMembershipStore(t *testing.T) {
 		rst.Recovery(d)
 		rc := &RaftCluster{lg: zaptest.NewLogger(t), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
 		rc.SetBackend(c.be)
-		rc.SetStore(rst)
+		//rc.SetStore(rst)
 		rc.Recover(func(lg *zap.Logger, v *semver.Version) { return })
 
 		//membership should match
@@ -1022,3 +1022,4 @@ func TestMembershipStore(t *testing.T) {
 		}
 	}
 }
+*/
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 5f4dc27791c..682042f45b0 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -179,6 +179,9 @@ func TestDoBadLocalAction(t *testing.T) {
 	}
 }
 
+/*
+Following test case crafts a V2 request and tests for repeat apply operation.
+This probably needs to be rewritten to apply for some request type compatible with v3 applier
 // TestApplyRepeat tests that server handles repeat raft messages gracefully
 func TestApplyRepeat(t *testing.T) {
 	n := newNodeConfChangeCommitterStream()
@@ -187,7 +190,7 @@ func TestApplyRepeat(t *testing.T) {
 	}
 	cl := newTestCluster(t, nil)
 	st := v2store.New()
-	cl.SetStore(v2store.New())
+	//cl.SetStore(v2store.New())
 	cl.AddMember(&membership.Member{ID: 1234}, true)
 	r := newRaftNode(raftNodeConfig{
 		lg:          zaptest.NewLogger(t),
@@ -245,6 +248,7 @@ func TestApplyRepeat(t *testing.T) {
 		t.Fatalf("error on stop (%v)", err)
 	}
 }
+*/
 
 func TestApplyRequest(t *testing.T) {
 	tests := []struct {
@@ -1309,6 +1313,8 @@ func TestTriggerSnap(t *testing.T) {
 	srv.Stop()
 }
 
+/*
+Following test case possibly needs to be rewritten to send out req type compatible with v3
 // TestConcurrentApplyAndSnapshotV3 will send out snapshots concurrently with
 // proposals.
 func TestConcurrentApplyAndSnapshotV3(t *testing.T) {
@@ -1321,7 +1327,7 @@ func TestConcurrentApplyAndSnapshotV3(t *testing.T) {
 	n := newNopReadyNode()
 	st := v2store.New()
 	cl := membership.NewCluster(lg)
-	cl.SetStore(st)
+	//cl.SetStore(st)
 
 	testdir := t.TempDir()
 	if err := os.MkdirAll(testdir+"/member/snap", 0755); err != nil {
@@ -1408,6 +1414,7 @@ func TestConcurrentApplyAndSnapshotV3(t *testing.T) {
 		t.Errorf("outdated=%v, want 0", outdated)
 	}
 }
+*/
 
 // TestAddMember tests AddMember can propose and perform node addition.
 func TestAddMember(t *testing.T) {
diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go
index 91582495d00..732b1980e78 100644
--- a/tests/e2e/v2store_deprecation_test.go
+++ b/tests/e2e/v2store_deprecation_test.go
@@ -26,7 +26,6 @@ import (
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/assert"
-	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 
 	"go.etcd.io/etcd/api/v3/version"
@@ -344,12 +343,10 @@ func openSnap(data []byte) v2store.Store {
 
 func assertMembershipEqual(t testing.TB, firstStore v2store.Store, secondStore v2store.Store) {
 	rc1 := membership.NewCluster(zaptest.NewLogger(t))
-	rc1.SetStore(firstStore)
-	rc1.Recover(func(lg *zap.Logger, v *semver.Version) { return })
+	rc1.RecoverMembersFromStore(firstStore)
 
 	rc2 := membership.NewCluster(zaptest.NewLogger(t))
-	rc2.SetStore(secondStore)
-	rc2.Recover(func(lg *zap.Logger, v *semver.Version) { return })
+	rc2.RecoverMembersFromStore(secondStore)
 
 	//membership should match
 	if g := rc1.Members(); !reflect.DeepEqual(g, rc2.Members()) {

From 7a1ae7306a6609ecec8fdf532b6914eaf6d0b472 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 20:31:58 +0000
Subject: [PATCH 26/44] Update backup command to not use SetStore

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 etcdutl/etcdutl/backup_command.go | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/etcdutl/etcdutl/backup_command.go b/etcdutl/etcdutl/backup_command.go
index 89121a37e95..115cc3a24d6 100644
--- a/etcdutl/etcdutl/backup_command.go
+++ b/etcdutl/etcdutl/backup_command.go
@@ -15,6 +15,7 @@
 package etcdutl
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"regexp"
@@ -28,9 +29,9 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
+	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
 	"go.etcd.io/etcd/server/v3/storage/schema"
@@ -178,21 +179,11 @@ func saveSnap(lg *zap.Logger, destSnap, srcSnap string, desired *desiredCluster)
 // mustTranslateV2store processes storeData such that they match 'desiredCluster'.
 // In particular the method overrides membership information.
 func mustTranslateV2store(lg *zap.Logger, storeData []byte, desired *desiredCluster) []byte {
-	st := v2store.New()
-	if err := st.Recovery(storeData); err != nil {
-		lg.Panic("cannot translate v2store", zap.Error(err))
-	}
-
 	raftCluster := membership.NewClusterFromMembers(lg, desired.clusterId, desired.members)
 	raftCluster.SetID(desired.nodeId, desired.clusterId)
-	raftCluster.SetStore(st)
-	raftCluster.PushMembershipToStorage()
-
-	outputData, err := st.Save()
-	if err != nil {
-		lg.Panic("cannot save v2store", zap.Error(err))
-	}
-	return outputData
+	d := etcdserver.GetMembershipInfoInV2Format(lg, raftCluster)
+	fmt.Printf("storeData = %v d = %v\n", storeData, d)
+	return d
 }
 
 func translateWAL(lg *zap.Logger, srcWAL string, walsnap walpb.Snapshot) (etcdserverpb.Metadata, raftpb.HardState, []raftpb.Entry) {

From 1bcd8c6b66bc493f9c301c7c1307e1a8a98fa607 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 20:40:03 +0000
Subject: [PATCH 27/44] Stop recovering membership info from v2store data

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go | 17 -----------------
 server/etcdserver/server.go    | 13 -------------
 2 files changed, 30 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index de5828c9439..655c5db1224 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -394,23 +394,6 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 	}
 
 	if snapshot != nil {
-		if cfg.V2Deprecation != config.V2_DEPR_2_GONE {
-			if err = st.Recovery(snapshot.Data); err != nil {
-				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
-			}
-
-			if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
-				cfg.Logger.Error("illegal v2store content", zap.Error(err))
-				return nil, be, err
-			}
-
-			cfg.Logger.Info(
-				"recovered v2 store from snapshot",
-				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
-				zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))),
-			)
-		}
-
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
 		}
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 1b3fa49d6e0..8f04db4f795 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1046,19 +1046,6 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) {
 		lg.Info("restored auth store")
 	}
 
-	if s.Cfg.V2Deprecation != config.V2_DEPR_2_GONE {
-		lg.Info("restoring v2 store")
-		if err := s.v2store.Recovery(toApply.snapshot.Data); err != nil {
-			lg.Panic("failed to restore v2 store", zap.Error(err))
-		}
-
-		if err := serverstorage.AssertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil {
-			lg.Panic("illegal v2store content", zap.Error(err))
-		}
-
-		lg.Info("restored v2 store")
-	}
-
 	s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe))
 
 	lg.Info("restoring cluster configuration")

From f7b83bd7a8899f762127f1e825cd2e3dab5b4e71 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 22:35:54 +0000
Subject: [PATCH 28/44] Disable v2 specific test. Revisit to rewrite for v3

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 682042f45b0..23237b7f5d2 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -493,7 +493,9 @@ func TestApplyRequest(t *testing.T) {
 	}
 }
 
+/*
 func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
+	//TODO geetasg rewrite for applierV3
 	cl := newTestCluster(t, []*membership.Member{{ID: 1}})
 	srv := &EtcdServer{
 		lgMu:    new(sync.RWMutex),
@@ -515,6 +517,7 @@ func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
 		t.Errorf("attributes = %v, want %v", g, w)
 	}
 }
+*/
 
 func TestApplyConfChangeError(t *testing.T) {
 	lg := zaptest.NewLogger(t)

From 338f747f20cdbaf55f6cfe3e0a6873441ead0320 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Wed, 31 May 2023 23:28:53 +0000
Subject: [PATCH 29/44] Allow v2deprecation mode of write-only

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/config/v2_deprecation.go       |  3 ++-
 server/etcdserver/bootstrap.go        |  4 ++++
 server/etcdserver/bootstrap_test.go   |  1 +
 server/etcdserver/server.go           |  4 ++++
 server/etcdserver/server_test.go      |  2 +-
 server/storage/util.go                | 12 +++++++++---
 tests/e2e/v2store_deprecation_test.go |  9 ++++++---
 7 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/server/config/v2_deprecation.go b/server/config/v2_deprecation.go
index c1bf8e41bf7..095209da667 100644
--- a/server/config/v2_deprecation.go
+++ b/server/config/v2_deprecation.go
@@ -27,7 +27,8 @@ const (
 	// V2store is neither written nor read for 3.6. v2snapshot is published only for backward compatibility
 	V2_DEPR_2_GONE = V2DeprecationEnum("gone")
 
-	V2_DEPR_DEFAULT = V2_DEPR_2_GONE
+	//TODO geetasg does thie default need to change
+	V2_DEPR_DEFAULT = V2_DEPR_1_WRITE_ONLY
 )
 
 func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool {
diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index 655c5db1224..e3cd769189c 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -394,6 +394,10 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe
 	}
 
 	if snapshot != nil {
+		if err := serverstorage.AssertV2DeprecationStage(cfg.Logger, cfg.V2Deprecation); err != nil {
+			cfg.Logger.Panic("illegal v2store content", zap.Error(err))
+		}
+
 		if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil {
 			cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err))
 		}
diff --git a/server/etcdserver/bootstrap_test.go b/server/etcdserver/bootstrap_test.go
index 55a20684fe8..37bc5c7081a 100644
--- a/server/etcdserver/bootstrap_test.go
+++ b/server/etcdserver/bootstrap_test.go
@@ -187,6 +187,7 @@ func TestBootstrapBackend(t *testing.T) {
 				DataDir:             dataDir,
 				BackendFreelistType: bolt.FreelistArrayType,
 				Logger:              zaptest.NewLogger(t),
+				V2Deprecation:       config.V2_DEPR_DEFAULT,
 			}
 
 			if tt.prepareData != nil {
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 8f04db4f795..b19964288a7 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1046,6 +1046,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) {
 		lg.Info("restored auth store")
 	}
 
+	if err := serverstorage.AssertV2DeprecationStage(lg, s.Cfg.V2Deprecation); err != nil {
+		lg.Panic("illegal v2store content", zap.Error(err))
+	}
+
 	s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe))
 
 	lg.Info("restoring cluster configuration")
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 23237b7f5d2..93265007b00 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1195,7 +1195,7 @@ func TestSnapshotOrdering(t *testing.T) {
 	s := &EtcdServer{
 		lgMu:         new(sync.RWMutex),
 		lg:           lg,
-		Cfg:          config.ServerConfig{Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, V2Deprecation: config.V2_DEPR_2_GONE},
+		Cfg:          config.ServerConfig{Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, V2Deprecation: config.V2_DEPR_DEFAULT},
 		r:            *r,
 		snapshotter:  snap.New(lg, snapdir),
 		cluster:      cl,
diff --git a/server/storage/util.go b/server/storage/util.go
index e1996cfe581..8f5d0821a4d 100644
--- a/server/storage/util.go
+++ b/server/storage/util.go
@@ -32,6 +32,7 @@ import (
 // AssertNoV2StoreContent -> depending on the deprecation stage, warns or report an error
 // if the v2store contains custom content.
 func AssertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error {
+	//TODO remove this method.
 	metaOnly, err := membership.IsMetaStoreOnly(st)
 	if err != nil {
 		return err
@@ -39,10 +40,15 @@ func AssertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage c
 	if metaOnly {
 		return nil
 	}
-	if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
-		return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage)
+	return AssertV2DeprecationStage(lg, deprecationStage)
+}
+
+func AssertV2DeprecationStage(lg *zap.Logger, deprecationStage config.V2DeprecationEnum) error {
+	//TODO geetasg should the write-only be supported? Update TestV2DeprecationFlags accordingly
+	//supported stages are "write-only", "write-only-drop-data" and "gone"
+	if !deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+		return fmt.Errorf("Unsupported stage --v2-deprecation=%s", deprecationStage)
 	}
-	lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.")
 	return nil
 }
 
diff --git a/tests/e2e/v2store_deprecation_test.go b/tests/e2e/v2store_deprecation_test.go
index 732b1980e78..0cfbbcfb409 100644
--- a/tests/e2e/v2store_deprecation_test.go
+++ b/tests/e2e/v2store_deprecation_test.go
@@ -101,9 +101,12 @@ func TestV2DeprecationFlags(t *testing.T) {
 		assertVerifyCannotStartV2deprecationNotYet(t, memberDataDir)
 	})
 
-	t.Run("--v2-deprecation=write-only fails", func(t *testing.T) {
-		assertVerifyCannotStartV2deprecationWriteOnly(t, memberDataDir)
-	})
+	//It is ok to start with write-only in 3.6
+	/*
+		t.Run("--v2-deprecation=write-only fails", func(t *testing.T) {
+			assertVerifyCannotStartV2deprecationWriteOnly(t, memberDataDir)
+		})
+	*/
 
 }
 

From 6e51502a01304c7800c451b774708c55e9685558 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Thu, 1 Jun 2023 06:03:56 +0000
Subject: [PATCH 30/44] Remove SetStore method on RaftCluster

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index f7fe1842690..aed011fc877 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -245,8 +245,6 @@ func (c *RaftCluster) SetID(localID, cid types.ID) {
 	c.buildMembershipMetric()
 }
 
-func (c *RaftCluster) SetStore(st v2store.Store) { c.v2store = st }
-
 func (c *RaftCluster) SetBackend(be MembershipBackend) {
 	c.be = be
 	c.be.MustCreateBackendBuckets()

From 96056364bdf94b410966a5f36f57eb0a003860f5 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Thu, 1 Jun 2023 17:17:24 +0000
Subject: [PATCH 31/44] Update RemoveMember func to only use be

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index aed011fc877..baca2d27311 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -424,10 +424,7 @@ func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) {
 func (c *RaftCluster) RemoveMember(id types.ID, shouldApplyV3 ShouldApplyV3) {
 	c.Lock()
 	defer c.Unlock()
-	if c.v2store != nil {
-		mustDeleteMemberFromStore(c.lg, c.v2store, id)
-	}
-	if c.be != nil && shouldApplyV3 {
+	if shouldApplyV3 {
 		c.be.MustDeleteMemberFromBackend(id)
 	}
 

From d9f149c0243ad0be311e01c3dce6f4e8a3f92644 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 00:15:37 +0000
Subject: [PATCH 32/44] Update function signature for recoverSnapshot

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index e3cd769189c..8f14874751b 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -221,8 +221,7 @@ func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, st v2store.Store, s
 	// TODO(serathius): Implement schema setup in fresh storage
 	var snapshot *raftpb.Snapshot
 	if haveWAL {
-		//geetasg TODO input st is empty - remove this step for v2deprecation and replace with recoverying from be
-		snapshot, be, err = recoverSnapshot(cfg, st, be, beExist, beHooks, ci, ss)
+		snapshot, be, err = recoverSnapshot(cfg, be, beExist, beHooks, ci, ss)
 		if err != nil {
 			return nil, err
 		}
@@ -380,7 +379,7 @@ func bootstrapClusterWithWAL(cfg config.ServerConfig, meta *snapshotMetadata) (*
 	}, nil
 }
 
-func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) {
+func recoverSnapshot(cfg config.ServerConfig, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) {
 	// Find a snapshot to start/restart a raft node
 	walSnaps, err := wal.ValidSnapshotEntries(cfg.Logger, cfg.WALDir())
 	if err != nil {

From 8dcc34d5f2609e6d223ebfd58225a91d1a54a9be Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 02:21:41 +0000
Subject: [PATCH 33/44] Remove v2store from RaftCluster

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/api/membership/cluster.go | 57 +++++----------------
 1 file changed, 12 insertions(+), 45 deletions(-)

diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index baca2d27311..009a45c1f32 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -46,8 +46,7 @@ type RaftCluster struct {
 	localID types.ID
 	cid     types.ID
 
-	v2store v2store.Store
-	be      MembershipBackend
+	be MembershipBackend
 
 	sync.Mutex // guards the fields below
 	version    *semver.Version
@@ -258,13 +257,8 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 	c.Lock()
 	defer c.Unlock()
 
-	if c.be != nil {
-		c.version = c.be.ClusterVersionFromBackend()
-		c.members, c.removed = c.be.MustReadMembersFromBackend()
-	} else {
-		c.version = clusterVersionFromStore(c.lg, c.v2store)
-		c.members, c.removed = membersFromStore(c.lg, c.v2store)
-	}
+	c.version = c.be.ClusterVersionFromBackend()
+	c.members, c.removed = c.be.MustReadMembersFromBackend()
 	c.buildMembershipMetric()
 
 	if c.be != nil {
@@ -305,11 +299,7 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange, shouldAp
 
 	var membersMap map[types.ID]*Member
 	var removedMap map[types.ID]bool
-	if c.v2store != nil {
-		membersMap, removedMap = membersFromStore(c.lg, c.v2store)
-	} else {
-		membersMap, removedMap = c.be.MustReadMembersFromBackend()
-	}
+	membersMap, removedMap = c.be.MustReadMembersFromBackend()
 	if !shouldApplyV3 {
 		//TODO geetasg - RaftCluster purely based on backend cannot validate an entry older than ci
 		return nil
@@ -399,10 +389,7 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange, shouldAp
 func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) {
 	c.Lock()
 	defer c.Unlock()
-	if c.v2store != nil {
-		mustSaveMemberToStore(c.lg, c.v2store, m)
-	}
-	if c.be != nil && shouldApplyV3 {
+	if shouldApplyV3 {
 		c.be.MustSaveMemberToBackend(m)
 	}
 
@@ -458,10 +445,7 @@ func (c *RaftCluster) UpdateAttributes(id types.ID, attr Attributes, shouldApply
 
 	if m, ok := c.members[id]; ok {
 		m.Attributes = attr
-		if c.v2store != nil {
-			mustUpdateMemberAttrInStore(c.lg, c.v2store, m)
-		}
-		if c.be != nil && shouldApplyV3 {
+		if shouldApplyV3 {
 			c.be.MustSaveMemberToBackend(m)
 		}
 		return
@@ -492,10 +476,7 @@ func (c *RaftCluster) PromoteMember(id types.ID, shouldApplyV3 ShouldApplyV3) {
 
 	c.members[id].RaftAttributes.IsLearner = false
 	c.updateMembershipMetric(id, true)
-	if c.v2store != nil {
-		mustUpdateMemberInStore(c.lg, c.v2store, c.members[id])
-	}
-	if c.be != nil && shouldApplyV3 {
+	if shouldApplyV3 {
 		c.be.MustSaveMemberToBackend(c.members[id])
 	}
 
@@ -511,10 +492,7 @@ func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes,
 	defer c.Unlock()
 
 	c.members[id].RaftAttributes = raftAttr
-	if c.v2store != nil {
-		mustUpdateMemberInStore(c.lg, c.v2store, c.members[id])
-	}
-	if c.be != nil && shouldApplyV3 {
+	if shouldApplyV3 {
 		c.be.MustSaveMemberToBackend(c.members[id])
 	}
 
@@ -560,10 +538,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *s
 	c.version = ver
 	sv := semver.Must(semver.NewVersion(version.Version))
 	serverversion.MustDetectDowngrade(c.lg, sv, c.version)
-	if c.v2store != nil {
-		mustSaveClusterVersionToStore(c.lg, c.v2store, ver)
-	}
-	if c.be != nil && shouldApplyV3 {
+	if shouldApplyV3 {
 		c.be.MustSaveClusterVersionToBackend(ver)
 	}
 	if oldVer != nil {
@@ -797,17 +772,9 @@ func (c *RaftCluster) VotingMemberIDs() []types.ID {
 // PushMembershipToStorage is overriding storage information about cluster's
 // members, such that they fully reflect internal RaftCluster's storage.
 func (c *RaftCluster) PushMembershipToStorage() {
-	if c.be != nil {
-		c.be.TrimMembershipFromBackend()
-		for _, m := range c.members {
-			c.be.MustSaveMemberToBackend(m)
-		}
-	}
-	if c.v2store != nil {
-		TrimMembershipFromV2Store(c.lg, c.v2store)
-		for _, m := range c.members {
-			mustSaveMemberToStore(c.lg, c.v2store, m)
-		}
+	c.be.TrimMembershipFromBackend()
+	for _, m := range c.members {
+		c.be.MustSaveMemberToBackend(m)
 	}
 }
 

From 2db3039745dc8ee1f81f26a35cccb1a0372b733c Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 04:49:00 +0000
Subject: [PATCH 34/44] Update function signature of bootstrapBackend

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go      | 4 ++--
 server/etcdserver/bootstrap_test.go | 4 +---
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index 8f14874751b..c534a3400e5 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -77,7 +77,7 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) {
 
 	haveWAL := wal.Exist(cfg.WALDir())
 	st := v2store.New(StoreClusterPrefix, StoreKeysPrefix)
-	backend, err := bootstrapBackend(cfg, haveWAL, st, ss)
+	backend, err := bootstrapBackend(cfg, haveWAL, ss)
 	if err != nil {
 		return nil, err
 	}
@@ -198,7 +198,7 @@ func bootstrapSnapshot(cfg config.ServerConfig) *snap.Snapshotter {
 	return snap.New(cfg.Logger, cfg.SnapDir())
 }
 
-func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, st v2store.Store, ss *snap.Snapshotter) (backend *bootstrappedBackend, err error) {
+func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, ss *snap.Snapshotter) (backend *bootstrappedBackend, err error) {
 	beExist := fileutil.Exist(cfg.BackendPath())
 	ci := cindex.NewConsistentIndex(nil)
 	beHooks := serverstorage.NewBackendHooks(cfg.Logger, ci)
diff --git a/server/etcdserver/bootstrap_test.go b/server/etcdserver/bootstrap_test.go
index 37bc5c7081a..8a04557c33e 100644
--- a/server/etcdserver/bootstrap_test.go
+++ b/server/etcdserver/bootstrap_test.go
@@ -41,7 +41,6 @@ import (
 	"go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	serverstorage "go.etcd.io/etcd/server/v3/storage"
 	"go.etcd.io/raft/v3/raftpb"
 )
@@ -197,9 +196,8 @@ func TestBootstrapBackend(t *testing.T) {
 			}
 
 			haveWAL := wal.Exist(cfg.WALDir())
-			st := v2store.New(StoreClusterPrefix, StoreKeysPrefix)
 			ss := snap.New(cfg.Logger, cfg.SnapDir())
-			backend, err := bootstrapBackend(cfg, haveWAL, st, ss)
+			backend, err := bootstrapBackend(cfg, haveWAL, ss)
 
 			hasError := err != nil
 			expectedHasError := tt.expectedError != nil

From 930ecd28bbd4dc63b89aff26e28e547ec5b056a6 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 18:52:34 +0000
Subject: [PATCH 35/44] Remove v2store from bootstrap

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/bootstrap.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go
index c534a3400e5..6cf7a8f19f3 100644
--- a/server/etcdserver/bootstrap.go
+++ b/server/etcdserver/bootstrap.go
@@ -38,7 +38,6 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v2discovery"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery"
 	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
 	servererrors "go.etcd.io/etcd/server/v3/etcdserver/errors"
@@ -76,7 +75,6 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) {
 	}
 
 	haveWAL := wal.Exist(cfg.WALDir())
-	st := v2store.New(StoreClusterPrefix, StoreKeysPrefix)
 	backend, err := bootstrapBackend(cfg, haveWAL, ss)
 	if err != nil {
 		return nil, err
@@ -96,7 +94,7 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) {
 		return nil, err
 	}
 
-	s, err := bootstrapStorage(cfg, st, backend, bwal, cluster)
+	s, err := bootstrapStorage(cfg, backend, bwal, cluster)
 	if err != nil {
 		backend.Close()
 		return nil, err
@@ -131,7 +129,6 @@ func (s *bootstrappedServer) Close() {
 type bootstrappedStorage struct {
 	backend *bootstrappedBackend
 	wal     *bootstrappedWAL
-	st      v2store.Store
 }
 
 func (s *bootstrappedStorage) Close() {
@@ -165,14 +162,13 @@ type bootstrappedRaft struct {
 	storage *raft.MemoryStorage
 }
 
-func bootstrapStorage(cfg config.ServerConfig, st v2store.Store, be *bootstrappedBackend, wal *bootstrappedWAL, cl *bootstrapedCluster) (b *bootstrappedStorage, err error) {
+func bootstrapStorage(cfg config.ServerConfig, be *bootstrappedBackend, wal *bootstrappedWAL, cl *bootstrapedCluster) (b *bootstrappedStorage, err error) {
 	if wal == nil {
 		wal = bootstrapNewWAL(cfg, cl)
 	}
 
 	return &bootstrappedStorage{
 		backend: be,
-		st:      st,
 		wal:     wal,
 	}, nil
 }

From 9b0dd9760a81a5b9d8a6ed009f3d668dc7350ef5 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 20:27:41 +0000
Subject: [PATCH 36/44] Use new v2store for etcdserver. Bootstrap does not
 return a store

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index b19964288a7..554984a0d81 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -325,7 +325,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 		lgMu:                  new(sync.RWMutex),
 		lg:                    cfg.Logger,
 		errorc:                make(chan error, 1),
-		v2store:               b.storage.st,
+		v2store:               v2store.New(StoreClusterPrefix, StoreKeysPrefix),
 		snapshotter:           b.ss,
 		r:                     *b.raft.newRaftNode(b.ss, b.storage.wal.w, b.cluster.cl),
 		memberId:              b.cluster.nodeID,

From 448f9ac3f5273ee2bc402016f2c2a9b8d906242b Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 21:36:15 +0000
Subject: [PATCH 37/44] Stop using v2store and v2applier from prod path

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 554984a0d81..71ee6b81804 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -325,7 +325,6 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 		lgMu:                  new(sync.RWMutex),
 		lg:                    cfg.Logger,
 		errorc:                make(chan error, 1),
-		v2store:               v2store.New(StoreClusterPrefix, StoreKeysPrefix),
 		snapshotter:           b.ss,
 		r:                     *b.raft.newRaftNode(b.ss, b.storage.wal.w, b.cluster.cl),
 		memberId:              b.cluster.nodeID,
@@ -343,7 +342,9 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	}
 	serverID.With(prometheus.Labels{"server_id": b.cluster.nodeID.String()}).Set(1)
 	srv.cluster.SetVersionChangedNotifier(srv.clusterVersionChanged)
-	srv.applyV2 = NewApplierV2(cfg.Logger, srv.v2store, srv.cluster)
+	if srv.v2store != nil {
+		srv.applyV2 = NewApplierV2(cfg.Logger, srv.v2store, srv.cluster)
+	}
 
 	srv.be = b.storage.backend.be
 	srv.beHooks = b.storage.backend.beHooks
@@ -440,6 +441,10 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	}
 	srv.r.transport = tr
 
+	if srv.v2store != nil {
+		panic(fmt.Sprintf("This constructor should not have set v2store"))
+	}
+
 	return srv, nil
 }
 
@@ -853,7 +858,7 @@ func (s *EtcdServer) run() {
 			lg.Warn("data-dir used by this member must be removed")
 			return
 		case <-getSyncC():
-			if s.v2store.HasTTLKeys() {
+			if s.v2store != nil && s.v2store.HasTTLKeys() {
 				s.sync(s.Cfg.ReqTimeout())
 			}
 		case <-s.stop:
@@ -1890,14 +1895,20 @@ func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry) {
 		rp := &r
 		pbutil.MustUnmarshal(rp, e.Data)
 		s.lg.Debug("applyEntryNormal", zap.Stringer("V2request", rp))
-		s.w.Trigger(r.ID, s.applyV2Request((*RequestV2)(rp), shouldApplyV3))
+		if s.v2store != nil {
+			s.lg.Debug("V2request applyEntryNormal", zap.Stringer("raftReq", &raftReq))
+			s.w.Trigger(r.ID, s.applyV2Request((*RequestV2)(rp), shouldApplyV3))
+		}
 		return
 	}
 	s.lg.Debug("applyEntryNormal", zap.Stringer("raftReq", &raftReq))
 
 	if raftReq.V2 != nil {
 		req := (*RequestV2)(raftReq.V2)
-		s.w.Trigger(req.ID, s.applyV2Request(req, shouldApplyV3))
+		s.lg.Debug("V2 applyEntryNormal", zap.Stringer("raftReq", &raftReq))
+		if s.v2store != nil {
+			s.w.Trigger(req.ID, s.applyV2Request(req, shouldApplyV3))
+		}
 		return
 	}
 

From 9c02d763d4b9859b7a648f3fd1050d126e84fcc1 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Fri, 2 Jun 2023 22:14:42 +0000
Subject: [PATCH 38/44] Try out removing v2 applier

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/apply_v2.go    | 166 ------------------------------
 server/etcdserver/server.go      |  87 ++--------------
 server/etcdserver/server_test.go |  22 ++--
 server/etcdserver/v2_server.go   | 167 -------------------------------
 4 files changed, 19 insertions(+), 423 deletions(-)
 delete mode 100644 server/etcdserver/apply_v2.go
 delete mode 100644 server/etcdserver/v2_server.go

diff --git a/server/etcdserver/apply_v2.go b/server/etcdserver/apply_v2.go
deleted file mode 100644
index c9e4c3e87b0..00000000000
--- a/server/etcdserver/apply_v2.go
+++ /dev/null
@@ -1,166 +0,0 @@
-// Copyright 2016 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package etcdserver
-
-import (
-	"encoding/json"
-	"fmt"
-	"path"
-	"time"
-	"unicode/utf8"
-
-	"github.com/coreos/go-semver/semver"
-
-	"go.etcd.io/etcd/pkg/v3/pbutil"
-	"go.etcd.io/etcd/server/v3/etcdserver/api"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
-	"go.etcd.io/etcd/server/v3/etcdserver/errors"
-	"go.etcd.io/etcd/server/v3/etcdserver/txn"
-
-	"go.uber.org/zap"
-)
-
-const v2Version = "v2"
-
-// ApplierV2 is the interface for processing V2 raft messages
-type ApplierV2 interface {
-	Delete(r *RequestV2) Response
-	Post(r *RequestV2) Response
-	Put(r *RequestV2, shouldApplyV3 membership.ShouldApplyV3) Response
-	QGet(r *RequestV2) Response
-	Sync(r *RequestV2) Response
-}
-
-func NewApplierV2(lg *zap.Logger, s v2store.Store, c *membership.RaftCluster) ApplierV2 {
-	if lg == nil {
-		lg = zap.NewNop()
-	}
-	return &applierV2store{lg: lg, store: s, cluster: c}
-}
-
-type applierV2store struct {
-	lg      *zap.Logger
-	store   v2store.Store
-	cluster *membership.RaftCluster
-}
-
-func (a *applierV2store) Delete(r *RequestV2) Response {
-	switch {
-	case r.PrevIndex > 0 || r.PrevValue != "":
-		return toResponse(a.store.CompareAndDelete(r.Path, r.PrevValue, r.PrevIndex))
-	default:
-		return toResponse(a.store.Delete(r.Path, r.Dir, r.Recursive))
-	}
-}
-
-func (a *applierV2store) Post(r *RequestV2) Response {
-	return toResponse(a.store.Create(r.Path, r.Dir, r.Val, true, r.TTLOptions()))
-}
-
-func (a *applierV2store) Put(r *RequestV2, shouldApplyV3 membership.ShouldApplyV3) Response {
-	ttlOptions := r.TTLOptions()
-	exists, existsSet := pbutil.GetBool(r.PrevExist)
-	switch {
-	case existsSet:
-		if exists {
-			if r.PrevIndex == 0 && r.PrevValue == "" {
-				return toResponse(a.store.Update(r.Path, r.Val, ttlOptions))
-			}
-			return toResponse(a.store.CompareAndSwap(r.Path, r.PrevValue, r.PrevIndex, r.Val, ttlOptions))
-		}
-		return toResponse(a.store.Create(r.Path, r.Dir, r.Val, false, ttlOptions))
-	case r.PrevIndex > 0 || r.PrevValue != "":
-		return toResponse(a.store.CompareAndSwap(r.Path, r.PrevValue, r.PrevIndex, r.Val, ttlOptions))
-	default:
-		if storeMemberAttributeRegexp.MatchString(r.Path) {
-			id := membership.MustParseMemberIDFromKey(a.lg, path.Dir(r.Path))
-			var attr membership.Attributes
-			if err := json.Unmarshal([]byte(r.Val), &attr); err != nil {
-				a.lg.Panic("failed to unmarshal", zap.String("value", r.Val), zap.Error(err))
-			}
-			if a.cluster != nil {
-				a.cluster.UpdateAttributes(id, attr, shouldApplyV3)
-			}
-			// return an empty response since there is no consumer.
-			return Response{}
-		}
-		// TODO remove v2 version set to avoid the conflict between v2 and v3 in etcd 3.6
-		if r.Path == membership.StoreClusterVersionKey() {
-			if a.cluster != nil {
-				// persist to backend given v2store can be very stale
-				a.cluster.SetVersion(semver.Must(semver.NewVersion(r.Val)), api.UpdateCapability, shouldApplyV3)
-			}
-			return Response{}
-		}
-		return toResponse(a.store.Set(r.Path, r.Dir, r.Val, ttlOptions))
-	}
-}
-
-func (a *applierV2store) QGet(r *RequestV2) Response {
-	return toResponse(a.store.Get(r.Path, r.Recursive, r.Sorted))
-}
-
-func (a *applierV2store) Sync(r *RequestV2) Response {
-	a.store.DeleteExpiredKeys(time.Unix(0, r.Time))
-	return Response{}
-}
-
-// applyV2Request interprets r as a call to v2store.X
-// and returns a Response interpreted from v2store.Event
-func (s *EtcdServer) applyV2Request(r *RequestV2, shouldApplyV3 membership.ShouldApplyV3) (resp Response) {
-	stringer := panicAlternativeStringer{
-		stringer:    r,
-		alternative: func() string { return fmt.Sprintf("id:%d,method:%s,path:%s", r.ID, r.Method, r.Path) },
-	}
-	defer func(start time.Time) {
-		if !utf8.ValidString(r.Method) {
-			s.lg.Info("method is not valid utf-8")
-			return
-		}
-		success := resp.Err == nil
-		txn.ApplySecObserve(v2Version, r.Method, success, time.Since(start))
-		txn.WarnOfExpensiveRequest(s.Logger(), s.Cfg.WarningApplyDuration, start, stringer, nil, nil)
-	}(time.Now())
-
-	switch r.Method {
-	case "POST":
-		return s.applyV2.Post(r)
-	case "PUT":
-		return s.applyV2.Put(r, shouldApplyV3)
-	case "DELETE":
-		return s.applyV2.Delete(r)
-	case "QGET":
-		return s.applyV2.QGet(r)
-	case "SYNC":
-		return s.applyV2.Sync(r)
-	default:
-		// This should never be reached, but just in case:
-		return Response{Err: errors.ErrUnknownMethod}
-	}
-}
-
-func (r *RequestV2) TTLOptions() v2store.TTLOptionSet {
-	refresh, _ := pbutil.GetBool(r.Refresh)
-	ttlOptions := v2store.TTLOptionSet{Refresh: refresh}
-	if r.Expiration != 0 {
-		ttlOptions.ExpireTime = time.Unix(0, r.Expiration)
-	}
-	return ttlOptions
-}
-
-func toResponse(ev *v2store.Event, err error) Response {
-	return Response{Event: ev, Err: err}
-}
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 71ee6b81804..9bed8e57c30 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -61,7 +61,6 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
 	stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor"
 	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
@@ -134,20 +133,11 @@ func init() {
 }
 
 type Response struct {
-	Term    uint64
-	Index   uint64
-	Event   *v2store.Event
-	Watcher v2store.Watcher
-	Err     error
-}
-
-type ServerV2 interface {
-	Server
-	Leader() types.ID
-
-	// Do takes a V2 request and attempts to fulfill it, returning a Response.
-	Do(ctx context.Context, r pb.Request) (Response, error)
-	ClientCertAuthEnabled() bool
+	Term  uint64
+	Index uint64
+	//Event   *v2store.Event
+	//Watcher v2store.Watcher
+	Err error
 }
 
 type ServerV3 interface {
@@ -249,11 +239,8 @@ type EtcdServer struct {
 
 	cluster *membership.RaftCluster
 
-	v2store     v2store.Store
 	snapshotter *snap.Snapshotter
 
-	applyV2 ApplierV2
-
 	uberApply apply.UberApplier
 
 	applyWait wait.WaitTime
@@ -342,9 +329,6 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	}
 	serverID.With(prometheus.Labels{"server_id": b.cluster.nodeID.String()}).Set(1)
 	srv.cluster.SetVersionChangedNotifier(srv.clusterVersionChanged)
-	if srv.v2store != nil {
-		srv.applyV2 = NewApplierV2(cfg.Logger, srv.v2store, srv.cluster)
-	}
 
 	srv.be = b.storage.backend.be
 	srv.beHooks = b.storage.backend.beHooks
@@ -441,10 +425,6 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	}
 	srv.r.transport = tr
 
-	if srv.v2store != nil {
-		panic(fmt.Sprintf("This constructor should not have set v2store"))
-	}
-
 	return srv, nil
 }
 
@@ -636,7 +616,7 @@ func (s *EtcdServer) Cluster() api.Cluster { return s.cluster }
 func (s *EtcdServer) ApplyWait() <-chan struct{} { return s.applyWait.Wait(s.getCommittedIndex()) }
 
 type ServerPeer interface {
-	ServerV2
+	Server
 	RaftHandler() http.Handler
 	LeaseHandler() http.Handler
 }
@@ -858,9 +838,7 @@ func (s *EtcdServer) run() {
 			lg.Warn("data-dir used by this member must be removed")
 			return
 		case <-getSyncC():
-			if s.v2store != nil && s.v2store.HasTTLKeys() {
-				s.sync(s.Cfg.ReqTimeout())
-			}
+			lg.Warn("NOP")
 		case <-s.stop:
 			return
 		}
@@ -1895,20 +1873,13 @@ func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry) {
 		rp := &r
 		pbutil.MustUnmarshal(rp, e.Data)
 		s.lg.Debug("applyEntryNormal", zap.Stringer("V2request", rp))
-		if s.v2store != nil {
-			s.lg.Debug("V2request applyEntryNormal", zap.Stringer("raftReq", &raftReq))
-			s.w.Trigger(r.ID, s.applyV2Request((*RequestV2)(rp), shouldApplyV3))
-		}
+		s.lg.Warn("V2request applyEntryNormal will be dropped", zap.Stringer("raftReq", &raftReq))
 		return
 	}
 	s.lg.Debug("applyEntryNormal", zap.Stringer("raftReq", &raftReq))
 
 	if raftReq.V2 != nil {
-		req := (*RequestV2)(raftReq.V2)
-		s.lg.Debug("V2 applyEntryNormal", zap.Stringer("raftReq", &raftReq))
-		if s.v2store != nil {
-			s.w.Trigger(req.ID, s.applyV2Request(req, shouldApplyV3))
-		}
+		s.lg.Warn("V2 applyEntryNormal will be dropped", zap.Stringer("raftReq", &raftReq))
 		return
 	}
 
@@ -2264,46 +2235,6 @@ func (s *EtcdServer) monitorCompactHash() {
 	}
 }
 
-func (s *EtcdServer) updateClusterVersionV2(ver string) {
-	lg := s.Logger()
-
-	if s.cluster.Version() == nil {
-		lg.Info(
-			"setting up initial cluster version using v2 API",
-			zap.String("cluster-version", version.Cluster(ver)),
-		)
-	} else {
-		lg.Info(
-			"updating cluster version using v2 API",
-			zap.String("from", version.Cluster(s.cluster.Version().String())),
-			zap.String("to", version.Cluster(ver)),
-		)
-	}
-
-	req := pb.Request{
-		Method: "PUT",
-		Path:   membership.StoreClusterVersionKey(),
-		Val:    ver,
-	}
-
-	ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout())
-	_, err := s.Do(ctx, req)
-	cancel()
-
-	switch err {
-	case nil:
-		lg.Info("cluster version is updated", zap.String("cluster-version", version.Cluster(ver)))
-		return
-
-	case errors.ErrStopped:
-		lg.Warn("aborting cluster version update; server is stopped", zap.Error(err))
-		return
-
-	default:
-		lg.Warn("failed to update cluster version", zap.Error(err))
-	}
-}
-
 func (s *EtcdServer) updateClusterVersionV3(ver string) {
 	lg := s.Logger()
 
diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 93265007b00..18dae0a4e72 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -20,9 +20,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"os"
-	"path"
-	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
@@ -34,10 +31,8 @@ import (
 
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/api/v3/membershippb"
-	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
 	"go.etcd.io/etcd/client/pkg/v3/types"
-	"go.etcd.io/etcd/client/pkg/v3/verify"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
 	"go.etcd.io/etcd/pkg/v3/wait"
@@ -46,17 +41,13 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	apply2 "go.etcd.io/etcd/server/v3/etcdserver/apply"
 	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
 	"go.etcd.io/etcd/server/v3/etcdserver/errors"
-	"go.etcd.io/etcd/server/v3/lease"
 	"go.etcd.io/etcd/server/v3/mock/mockstorage"
-	"go.etcd.io/etcd/server/v3/mock/mockstore"
 	"go.etcd.io/etcd/server/v3/mock/mockwait"
 	serverstorage "go.etcd.io/etcd/server/v3/storage"
 	betesting "go.etcd.io/etcd/server/v3/storage/backend/testing"
-	"go.etcd.io/etcd/server/v3/storage/mvcc"
 	"go.etcd.io/etcd/server/v3/storage/schema"
 	"go.etcd.io/raft/v3"
 	"go.etcd.io/raft/v3/raftpb"
@@ -64,6 +55,7 @@ import (
 
 // TestDoLocalAction tests requests which do not need to go through raft to be applied,
 // and are served through local data.
+/*
 func TestDoLocalAction(t *testing.T) {
 	tests := []struct {
 		req pb.Request
@@ -123,9 +115,11 @@ func TestDoLocalAction(t *testing.T) {
 		}
 	}
 }
+*/
 
 // TestDoBadLocalAction tests server requests which do not need to go through consensus,
 // and return errors when they fetch from local data.
+/*
 func TestDoBadLocalAction(t *testing.T) {
 	storeErr := fmt.Errorf("bah")
 	tests := []struct {
@@ -178,6 +172,7 @@ func TestDoBadLocalAction(t *testing.T) {
 		}
 	}
 }
+*/
 
 /*
 Following test case crafts a V2 request and tests for repeat apply operation.
@@ -249,7 +244,7 @@ func TestApplyRepeat(t *testing.T) {
 	}
 }
 */
-
+/*
 func TestApplyRequest(t *testing.T) {
 	tests := []struct {
 		req pb.Request
@@ -492,6 +487,7 @@ func TestApplyRequest(t *testing.T) {
 		}
 	}
 }
+*/
 
 /*
 func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
@@ -790,6 +786,7 @@ func TestApplyMultiConfChangeShouldStop(t *testing.T) {
 	}
 }
 
+/*
 func TestDoProposal(t *testing.T) {
 	tests := []pb.Request{
 		{Method: "POST", ID: 1},
@@ -1316,7 +1313,6 @@ func TestTriggerSnap(t *testing.T) {
 	srv.Stop()
 }
 
-/*
 Following test case possibly needs to be rewritten to send out req type compatible with v3
 // TestConcurrentApplyAndSnapshotV3 will send out snapshots concurrently with
 // proposals.
@@ -1678,6 +1674,7 @@ func TestPublishV3Retry(t *testing.T) {
 	<-ch
 }
 
+/*
 func TestUpdateVersion(t *testing.T) {
 	n := newNodeRecorder()
 	ch := make(chan interface{}, 1)
@@ -1700,7 +1697,7 @@ func TestUpdateVersion(t *testing.T) {
 		ctx:    ctx,
 		cancel: cancel,
 	}
-	srv.updateClusterVersionV2("2.0.0")
+	srv.updateClusterVersionV3("2.0.0")
 
 	action := n.Action()
 	if len(action) != 1 {
@@ -1724,6 +1721,7 @@ func TestUpdateVersion(t *testing.T) {
 		t.Errorf("val = %s, want %s", r.Val, "2.0.0")
 	}
 }
+*/
 
 func TestStopNotify(t *testing.T) {
 	s := &EtcdServer{
diff --git a/server/etcdserver/v2_server.go b/server/etcdserver/v2_server.go
deleted file mode 100644
index 517d7ca7f70..00000000000
--- a/server/etcdserver/v2_server.go
+++ /dev/null
@@ -1,167 +0,0 @@
-// Copyright 2016 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package etcdserver
-
-import (
-	"context"
-	"time"
-
-	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
-	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
-	"go.etcd.io/etcd/server/v3/etcdserver/errors"
-)
-
-type RequestV2 pb.Request
-
-type RequestV2Handler interface {
-	Post(ctx context.Context, r *RequestV2) (Response, error)
-	Put(ctx context.Context, r *RequestV2) (Response, error)
-	Delete(ctx context.Context, r *RequestV2) (Response, error)
-	QGet(ctx context.Context, r *RequestV2) (Response, error)
-	Get(ctx context.Context, r *RequestV2) (Response, error)
-	Head(ctx context.Context, r *RequestV2) (Response, error)
-}
-
-type reqV2HandlerEtcdServer struct {
-	reqV2HandlerStore
-	s *EtcdServer
-}
-
-type reqV2HandlerStore struct {
-	store   v2store.Store
-	applier ApplierV2
-}
-
-func NewStoreRequestV2Handler(s v2store.Store, applier ApplierV2) RequestV2Handler {
-	return &reqV2HandlerStore{s, applier}
-}
-
-func (a *reqV2HandlerStore) Post(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.applier.Post(r), nil
-}
-
-func (a *reqV2HandlerStore) Put(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.applier.Put(r, membership.ApplyBoth), nil
-}
-
-func (a *reqV2HandlerStore) Delete(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.applier.Delete(r), nil
-}
-
-func (a *reqV2HandlerStore) QGet(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.applier.QGet(r), nil
-}
-
-func (a *reqV2HandlerStore) Get(ctx context.Context, r *RequestV2) (Response, error) {
-	if r.Wait {
-		wc, err := a.store.Watch(r.Path, r.Recursive, r.Stream, r.Since)
-		return Response{Watcher: wc}, err
-	}
-	ev, err := a.store.Get(r.Path, r.Recursive, r.Sorted)
-	return Response{Event: ev}, err
-}
-
-func (a *reqV2HandlerStore) Head(ctx context.Context, r *RequestV2) (Response, error) {
-	ev, err := a.store.Get(r.Path, r.Recursive, r.Sorted)
-	return Response{Event: ev}, err
-}
-
-func (a *reqV2HandlerEtcdServer) Post(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.processRaftRequest(ctx, r)
-}
-
-func (a *reqV2HandlerEtcdServer) Put(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.processRaftRequest(ctx, r)
-}
-
-func (a *reqV2HandlerEtcdServer) Delete(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.processRaftRequest(ctx, r)
-}
-
-func (a *reqV2HandlerEtcdServer) QGet(ctx context.Context, r *RequestV2) (Response, error) {
-	return a.processRaftRequest(ctx, r)
-}
-
-func (a *reqV2HandlerEtcdServer) processRaftRequest(ctx context.Context, r *RequestV2) (Response, error) {
-	data, err := ((*pb.Request)(r)).Marshal()
-	if err != nil {
-		return Response{}, err
-	}
-	ch := a.s.w.Register(r.ID)
-
-	start := time.Now()
-	a.s.r.Propose(ctx, data)
-	proposalsPending.Inc()
-	defer proposalsPending.Dec()
-
-	select {
-	case x := <-ch:
-		resp := x.(Response)
-		return resp, resp.Err
-	case <-ctx.Done():
-		proposalsFailed.Inc()
-		a.s.w.Trigger(r.ID, nil) // GC wait
-		return Response{}, a.s.parseProposeCtxErr(ctx.Err(), start)
-	case <-a.s.stopping:
-	}
-	return Response{}, errors.ErrStopped
-}
-
-func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
-	r.ID = s.reqIDGen.Next()
-	h := &reqV2HandlerEtcdServer{
-		reqV2HandlerStore: reqV2HandlerStore{
-			store:   s.v2store,
-			applier: s.applyV2,
-		},
-		s: s,
-	}
-	rp := &r
-	resp, err := ((*RequestV2)(rp)).Handle(ctx, h)
-	resp.Term, resp.Index = s.Term(), s.CommittedIndex()
-	return resp, err
-}
-
-// Handle interprets r and performs an operation on s.store according to r.Method
-// and other fields. If r.Method is "POST", "PUT", "DELETE", or a "GET" with
-// Quorum == true, r will be sent through consensus before performing its
-// respective operation. Do will block until an action is performed or there is
-// an error.
-func (r *RequestV2) Handle(ctx context.Context, v2api RequestV2Handler) (Response, error) {
-	if r.Method == "GET" && r.Quorum {
-		r.Method = "QGET"
-	}
-	switch r.Method {
-	case "POST":
-		return v2api.Post(ctx, r)
-	case "PUT":
-		return v2api.Put(ctx, r)
-	case "DELETE":
-		return v2api.Delete(ctx, r)
-	case "QGET":
-		return v2api.QGet(ctx, r)
-	case "GET":
-		return v2api.Get(ctx, r)
-	case "HEAD":
-		return v2api.Head(ctx, r)
-	}
-	return Response{}, errors.ErrUnknownMethod
-}
-
-func (r *RequestV2) String() string {
-	rpb := pb.Request(*r)
-	return rpb.String()
-}

From 733d109241bcf60c590fce9fd04f1e98d710dbfa Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 12 Jun 2023 01:19:37 +0000
Subject: [PATCH 39/44] Use v2 to v3 applier only to handle publish from 3.5
 Drop other v2 requests

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/apply_v2v3.go | 107 ++++++++++++++++++++++++++++++++
 server/etcdserver/server.go     |  11 +++-
 2 files changed, 115 insertions(+), 3 deletions(-)
 create mode 100644 server/etcdserver/apply_v2v3.go

diff --git a/server/etcdserver/apply_v2v3.go b/server/etcdserver/apply_v2v3.go
new file mode 100644
index 00000000000..2113538ba96
--- /dev/null
+++ b/server/etcdserver/apply_v2v3.go
@@ -0,0 +1,107 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdserver
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"time"
+	"unicode/utf8"
+
+	"github.com/coreos/go-semver/semver"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"go.etcd.io/etcd/server/v3/etcdserver/api"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
+	"go.etcd.io/etcd/server/v3/etcdserver/errors"
+	"go.etcd.io/etcd/server/v3/etcdserver/txn"
+
+	"go.uber.org/zap"
+)
+
+const v2Version = "v2"
+
+type RequestV2 pb.Request
+
+func (r *RequestV2) String() string {
+	rpb := pb.Request(*r)
+	return rpb.String()
+}
+
+// ApplierV2ToV3Backend is the interface for processing V2 raft messages and apply it to v3 backend. This is meant to handle publish requests from 3.5. To be removed in 3.7
+type ApplierV2ToV3 interface {
+	Put(r *RequestV2) Response
+}
+
+func NewApplierV2ToV3(lg *zap.Logger, c *membership.RaftCluster) ApplierV2ToV3 {
+	if lg == nil {
+		lg = zap.NewNop()
+	}
+	return &applierV2ToV3{lg: lg, cluster: c}
+}
+
+type applierV2ToV3 struct {
+	lg      *zap.Logger
+	cluster *membership.RaftCluster
+}
+
+func (s *EtcdServer) applyV2RequestToV3(r *RequestV2) (resp Response) {
+	stringer := panicAlternativeStringer{
+		stringer:    r,
+		alternative: func() string { return fmt.Sprintf("id:%d,method:%s,path:%s", r.ID, r.Method, r.Path) },
+	}
+	defer func(start time.Time) {
+		if !utf8.ValidString(r.Method) {
+			s.lg.Info("method is not valid utf-8")
+			return
+		}
+		success := resp.Err == nil
+		txn.ApplySecObserve(v2Version, r.Method, success, time.Since(start))
+		txn.WarnOfExpensiveRequest(s.Logger(), s.Cfg.WarningApplyDuration, start, stringer, nil, nil)
+	}(time.Now())
+
+	switch r.Method {
+	case "PUT":
+		return s.applyV2ToV3.Put(r)
+	default:
+		// This should never be reached, but just in case:
+		return Response{Err: errors.ErrUnknownMethod}
+	}
+}
+
+func (a *applierV2ToV3) Put(r *RequestV2) Response {
+	if storeMemberAttributeRegexp.MatchString(r.Path) {
+		id := membership.MustParseMemberIDFromKey(a.lg, path.Dir(r.Path))
+		var attr membership.Attributes
+		if err := json.Unmarshal([]byte(r.Val), &attr); err != nil {
+			a.lg.Panic("failed to unmarshal", zap.String("value", r.Val), zap.Error(err))
+		}
+		if a.cluster != nil {
+			a.cluster.UpdateAttributes(id, attr, true)
+		}
+		// return an empty response since there is no consumer.
+		return Response{}
+	}
+	// TODO remove v2 version set to avoid the conflict between v2 and v3 in etcd 3.6
+	if r.Path == membership.StoreClusterVersionKey() {
+		if a.cluster != nil {
+			// persist to backend given v2store can be very stale
+			a.cluster.SetVersion(semver.Must(semver.NewVersion(r.Val)), api.UpdateCapability, true)
+		}
+		return Response{}
+	}
+	a.lg.Panic("unexpected v2 Put request")
+	return Response{}
+}
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 9bed8e57c30..61bb02d0bf9 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -241,6 +241,8 @@ type EtcdServer struct {
 
 	snapshotter *snap.Snapshotter
 
+	applyV2ToV3 ApplierV2ToV3
+
 	uberApply apply.UberApplier
 
 	applyWait wait.WaitTime
@@ -329,6 +331,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	}
 	serverID.With(prometheus.Labels{"server_id": b.cluster.nodeID.String()}).Set(1)
 	srv.cluster.SetVersionChangedNotifier(srv.clusterVersionChanged)
+	srv.applyV2ToV3 = NewApplierV2ToV3(cfg.Logger, srv.cluster)
 
 	srv.be = b.storage.backend.be
 	srv.beHooks = b.storage.backend.beHooks
@@ -1872,14 +1875,16 @@ func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry) {
 		var r pb.Request
 		rp := &r
 		pbutil.MustUnmarshal(rp, e.Data)
-		s.lg.Debug("applyEntryNormal", zap.Stringer("V2request", rp))
-		s.lg.Warn("V2request applyEntryNormal will be dropped", zap.Stringer("raftReq", &raftReq))
+		s.lg.Debug("applyEntryNormal", zap.Stringer("V2request handled", rp))
+		//TODO remove this for 3.7. This handles the publish request from 3.5 members.
+		s.w.Trigger(r.ID, s.applyV2RequestToV3((*RequestV2)(rp)))
 		return
 	}
 	s.lg.Debug("applyEntryNormal", zap.Stringer("raftReq", &raftReq))
 
 	if raftReq.V2 != nil {
-		s.lg.Warn("V2 applyEntryNormal will be dropped", zap.Stringer("raftReq", &raftReq))
+		req := (*RequestV2)(raftReq.V2)
+		s.lg.Debug("applyEntryNormal", zap.Stringer("V2request dropped", req))
 		return
 	}
 

From 1bd5ae62ece7029275a3e04a178a852d9eef754e Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 12 Jun 2023 04:16:54 +0000
Subject: [PATCH 40/44] Fix comment

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/apply_v2v3.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/server/etcdserver/apply_v2v3.go b/server/etcdserver/apply_v2v3.go
index 2113538ba96..8b86be60b7d 100644
--- a/server/etcdserver/apply_v2v3.go
+++ b/server/etcdserver/apply_v2v3.go
@@ -40,7 +40,6 @@ func (r *RequestV2) String() string {
 	return rpb.String()
 }
 
-// ApplierV2ToV3Backend is the interface for processing V2 raft messages and apply it to v3 backend. This is meant to handle publish requests from 3.5. To be removed in 3.7
 type ApplierV2ToV3 interface {
 	Put(r *RequestV2) Response
 }

From 21a1e442b8de004632e03602fc9bf3d63313eb02 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 12 Jun 2023 04:24:35 +0000
Subject: [PATCH 41/44] Fix comment

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 61bb02d0bf9..e62e944a8d3 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -135,9 +135,7 @@ func init() {
 type Response struct {
 	Term  uint64
 	Index uint64
-	//Event   *v2store.Event
-	//Watcher v2store.Watcher
-	Err error
+	Err   error
 }
 
 type ServerV3 interface {

From a301201ca8b644ddadcf6abef01007e6dfc6e65c Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 12 Jun 2023 04:29:56 +0000
Subject: [PATCH 42/44] Fix goimport

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/apply_v2v3.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/etcdserver/apply_v2v3.go b/server/etcdserver/apply_v2v3.go
index 8b86be60b7d..71905a5fb24 100644
--- a/server/etcdserver/apply_v2v3.go
+++ b/server/etcdserver/apply_v2v3.go
@@ -22,6 +22,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/coreos/go-semver/semver"
+
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/server/v3/etcdserver/api"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"

From 9515be1e0e42159d19825a76902c933d1cdb5ab5 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Mon, 12 Jun 2023 21:37:12 +0000
Subject: [PATCH 43/44] Add unit test for updateVersionV3

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 47 ++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 18dae0a4e72..8786f8717b0 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -1723,6 +1723,53 @@ func TestUpdateVersion(t *testing.T) {
 }
 */
 
+func TestUpdateVersion3(t *testing.T) {
+	n := newNodeRecorder()
+	ch := make(chan interface{}, 1)
+	// simulate that request has gone through consensus
+	ch <- &apply2.Result{}
+	w := wait.NewWithResponse(ch)
+	// simulate that request has gone through consensus
+	//ch <- Response{}
+	//w := wait.NewWithResponse(ch)
+	ctx, cancel := context.WithCancel(context.TODO())
+	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	srv := &EtcdServer{
+		lgMu:       new(sync.RWMutex),
+		lg:         zaptest.NewLogger(t),
+		memberId:   1,
+		Cfg:        config.ServerConfig{Logger: lg, TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, MaxRequestBytes: 1000},
+		r:          *newRaftNode(raftNodeConfig{lg: zaptest.NewLogger(t), Node: n}),
+		attributes: membership.Attributes{Name: "node1", ClientURLs: []string{"http://node1.com"}},
+		cluster:    &membership.RaftCluster{},
+		w:          w,
+		reqIDGen:   idutil.NewGenerator(0, time.Time{}),
+		SyncTicker: &time.Ticker{},
+		authStore:  auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 0),
+		be:         be,
+
+		ctx:    ctx,
+		cancel: cancel,
+	}
+	ver := "2.0.0"
+	srv.updateClusterVersionV3(ver)
+
+	action := n.Action()
+	if len(action) != 1 {
+		t.Fatalf("len(action) = %d, want 1", len(action))
+	}
+	if action[0].Name != "Propose" {
+		t.Fatalf("action = %s, want Propose", action[0].Name)
+	}
+	data := action[0].Params[0].([]byte)
+	var r pb.InternalRaftRequest
+	if err := r.Unmarshal(data); err != nil {
+		t.Fatalf("unmarshal request error: %v", err)
+	}
+	assert.Equal(t, &membershippb.ClusterVersionSetRequest{Ver: ver}, r.ClusterVersionSet)
+}
+
 func TestStopNotify(t *testing.T) {
 	s := &EtcdServer{
 		lgMu: new(sync.RWMutex),

From d2323b7b2cc3c944c3d2bb6ac086b721c8ecaf73 Mon Sep 17 00:00:00 2001
From: Geeta Gharpure <geetagh@amazon.com>
Date: Tue, 13 Jun 2023 18:08:30 +0000
Subject: [PATCH 44/44] Update TestApplyRequestOnAdminMemberAttributes to use
 v2v3 applier

Signed-off-by: Geeta Gharpure <geetagh@amazon.com>
---
 server/etcdserver/server_test.go | 33 ++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/server/etcdserver/server_test.go b/server/etcdserver/server_test.go
index 8786f8717b0..c8fa2950c65 100644
--- a/server/etcdserver/server_test.go
+++ b/server/etcdserver/server_test.go
@@ -47,6 +47,7 @@ import (
 	"go.etcd.io/etcd/server/v3/mock/mockstorage"
 	"go.etcd.io/etcd/server/v3/mock/mockwait"
 	serverstorage "go.etcd.io/etcd/server/v3/storage"
+	"go.etcd.io/etcd/server/v3/storage/backend"
 	betesting "go.etcd.io/etcd/server/v3/storage/backend/testing"
 	"go.etcd.io/etcd/server/v3/storage/schema"
 	"go.etcd.io/raft/v3"
@@ -183,7 +184,7 @@ func TestApplyRepeat(t *testing.T) {
 	n.readyc <- raft.Ready{
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
-	cl := newTestCluster(t, nil)
+	cl := newTestCluster(t, nil, nil)
 	st := v2store.New()
 	//cl.SetStore(v2store.New())
 	cl.AddMember(&membership.Member{ID: 1234}, true)
@@ -489,17 +490,18 @@ func TestApplyRequest(t *testing.T) {
 }
 */
 
-/*
 func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
-	//TODO geetasg rewrite for applierV3
-	cl := newTestCluster(t, []*membership.Member{{ID: 1}})
+	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+	cl := newTestCluster(t, be, []*membership.Member{{ID: 1}})
 	srv := &EtcdServer{
 		lgMu:    new(sync.RWMutex),
-		lg:      zaptest.NewLogger(t),
-		v2store: mockstore.NewRecorder(),
+		lg:      lg,
 		cluster: cl,
+		be:      be,
 	}
-	srv.applyV2 = &applierV2store{store: srv.v2store, cluster: srv.cluster}
+	srv.applyV2ToV3 = NewApplierV2ToV3(lg, srv.cluster)
 
 	req := pb.Request{
 		Method: "PUT",
@@ -507,13 +509,12 @@ func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
 		Path:   membership.MemberAttributesStorePath(1),
 		Val:    `{"Name":"abc","ClientURLs":["http://127.0.0.1:2379"]}`,
 	}
-	srv.applyV2Request((*RequestV2)(&req), membership.ApplyBoth)
+	srv.applyV2RequestToV3((*RequestV2)(&req))
 	w := membership.Attributes{Name: "abc", ClientURLs: []string{"http://127.0.0.1:2379"}}
 	if g := cl.Member(1).Attributes; !reflect.DeepEqual(g, w) {
 		t.Errorf("attributes = %v, want %v", g, w)
 	}
 }
-*/
 
 func TestApplyConfChangeError(t *testing.T) {
 	lg := zaptest.NewLogger(t)
@@ -1422,7 +1423,7 @@ func TestAddMember(t *testing.T) {
 	n.readyc <- raft.Ready{
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
-	cl := newTestCluster(t, nil)
+	cl := newTestCluster(t, nil, nil)
 	be, _ := betesting.NewDefaultTmpBackend(t)
 	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	r := newRaftNode(raftNodeConfig{
@@ -1467,7 +1468,7 @@ func TestRemoveMember(t *testing.T) {
 	n.readyc <- raft.Ready{
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
-	cl := newTestCluster(t, nil)
+	cl := newTestCluster(t, nil, nil)
 	be, _ := betesting.NewDefaultTmpBackend(t)
 	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: 1234}, true)
@@ -1512,7 +1513,7 @@ func TestUpdateMember(t *testing.T) {
 	n.readyc <- raft.Ready{
 		SoftState: &raft.SoftState{RaftState: raft.StateLeader},
 	}
-	cl := newTestCluster(t, nil)
+	cl := newTestCluster(t, nil, nil)
 	be, _ := betesting.NewDefaultTmpBackend(t)
 	cl.SetBackend(schema.NewMembershipBackend(lg, be))
 	cl.AddMember(&membership.Member{ID: 1234}, true)
@@ -1977,8 +1978,12 @@ func (n *nodeCommitter) Propose(ctx context.Context, data []byte) error {
 	return nil
 }
 
-func newTestCluster(t testing.TB, membs []*membership.Member) *membership.RaftCluster {
-	c := membership.NewCluster(zaptest.NewLogger(t))
+func newTestCluster(t testing.TB, be backend.Backend, membs []*membership.Member) *membership.RaftCluster {
+	lg := zaptest.NewLogger(t)
+	c := membership.NewCluster(lg)
+	if be != nil {
+		c.SetBackend(schema.NewMembershipBackend(lg, be))
+	}
 	for _, m := range membs {
 		c.AddMember(m, true)
 	}