From 3dec1cb144359211d906b97efa8985ea5c179f02 Mon Sep 17 00:00:00 2001 From: Geeta Gharpure Date: Thu, 24 Aug 2023 17:57:36 +0000 Subject: [PATCH] update bootstrap to not recover v2store --- server/config/v2_deprecation.go | 2 +- server/etcdserver/bootstrap.go | 29 ++++++++--------------------- server/etcdserver/server.go | 15 +++++---------- server/storage/util.go | 18 ++++-------------- 4 files changed, 18 insertions(+), 46 deletions(-) diff --git a/server/config/v2_deprecation.go b/server/config/v2_deprecation.go index 862c3bb9343..e2eea8c0dc3 100644 --- a/server/config/v2_deprecation.go +++ b/server/config/v2_deprecation.go @@ -28,7 +28,7 @@ const ( // ability to rollback to etcd v3.5. V2_DEPR_2_GONE = V2DeprecationEnum("gone") - V2_DEPR_DEFAULT = V2_DEPR_1_WRITE_ONLY + V2_DEPR_DEFAULT = V2_DEPR_1_WRITE_ONLY_DROP ) func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool { diff --git a/server/etcdserver/bootstrap.go b/server/etcdserver/bootstrap.go index e416fd079c2..739d392275b 100644 --- a/server/etcdserver/bootstrap.go +++ b/server/etcdserver/bootstrap.go @@ -76,8 +76,7 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) { } haveWAL := wal.Exist(cfg.WALDir()) - st := v2store.New(StoreClusterPrefix, StoreKeysPrefix) - backend, err := bootstrapBackend(cfg, haveWAL, st, ss) + backend, err := bootstrapBackend(cfg, haveWAL, ss) if err != nil { return nil, err } @@ -96,7 +95,7 @@ func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) { return nil, err } - s, err := bootstrapStorage(cfg, st, backend, bwal, cluster) + s, err := bootstrapStorage(cfg, backend, bwal, cluster) if err != nil { backend.Close() return nil, err @@ -165,14 +164,13 @@ type bootstrappedRaft struct { storage *raft.MemoryStorage } -func bootstrapStorage(cfg config.ServerConfig, st v2store.Store, be *bootstrappedBackend, wal *bootstrappedWAL, cl *bootstrapedCluster) (b *bootstrappedStorage, err error) { +func bootstrapStorage(cfg config.ServerConfig, be *bootstrappedBackend, wal *bootstrappedWAL, cl *bootstrapedCluster) (b *bootstrappedStorage, err error) { if wal == nil { wal = bootstrapNewWAL(cfg, cl) } return &bootstrappedStorage{ backend: be, - st: st, wal: wal, }, nil } @@ -198,7 +196,7 @@ func bootstrapSnapshot(cfg config.ServerConfig) *snap.Snapshotter { return snap.New(cfg.Logger, cfg.SnapDir()) } -func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, st v2store.Store, ss *snap.Snapshotter) (backend *bootstrappedBackend, err error) { +func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, ss *snap.Snapshotter) (backend *bootstrappedBackend, err error) { beExist := fileutil.Exist(cfg.BackendPath()) ci := cindex.NewConsistentIndex(nil) beHooks := serverstorage.NewBackendHooks(cfg.Logger, ci) @@ -221,7 +219,7 @@ func bootstrapBackend(cfg config.ServerConfig, haveWAL bool, st v2store.Store, s // TODO(serathius): Implement schema setup in fresh storage var snapshot *raftpb.Snapshot if haveWAL { - snapshot, be, err = recoverSnapshot(cfg, st, be, beExist, beHooks, ci, ss) + snapshot, be, err = recoverSnapshot(cfg, be, beExist, beHooks, ci, ss) if err != nil { return nil, err } @@ -378,7 +376,7 @@ func bootstrapClusterWithWAL(cfg config.ServerConfig, meta *snapshotMetadata) (* }, nil } -func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) { +func recoverSnapshot(cfg config.ServerConfig, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer, ss *snap.Snapshotter) (*raftpb.Snapshot, backend.Backend, error) { // Find a snapshot to start/restart a raft node walSnaps, err := wal.ValidSnapshotEntries(cfg.Logger, cfg.WALDir()) if err != nil { @@ -392,21 +390,11 @@ func recoverSnapshot(cfg config.ServerConfig, st v2store.Store, be backend.Backe } if snapshot != nil { - if err = st.Recovery(snapshot.Data); err != nil { - cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err)) - } - - if err = serverstorage.AssertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil { - cfg.Logger.Error("illegal v2store content", zap.Error(err)) + if err = serverstorage.AssertV2DeprecationStage(cfg.Logger, cfg.V2Deprecation); err != nil { + cfg.Logger.Error("illegal v2store deprecation stage. This version does not recover from v2 snapshot", zap.Error(err)) return nil, be, err } - cfg.Logger.Info( - "recovered v2 store from snapshot", - zap.Uint64("snapshot-index", snapshot.Metadata.Index), - zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))), - ) - if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil { cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err)) } @@ -446,7 +434,6 @@ func (c *bootstrapedCluster) Finalize(cfg config.ServerConfig, s *bootstrappedSt if !s.wal.haveWAL { c.cl.SetID(c.nodeID, c.cl.ID()) } - c.cl.SetStore(s.st) c.cl.SetBackend(schema.NewMembershipBackend(cfg.Logger, s.backend.be)) if s.wal.haveWAL { c.cl.Recover(api.UpdateCapability) diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index c73d4776985..d17fa975003 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -322,7 +322,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { lgMu: new(sync.RWMutex), lg: cfg.Logger, errorc: make(chan error, 1), - v2store: b.storage.st, + v2store: v2store.New(StoreClusterPrefix, StoreKeysPrefix), snapshotter: b.ss, r: *b.raft.newRaftNode(b.ss, b.storage.wal.w, b.cluster.cl), memberId: b.cluster.nodeID, @@ -340,6 +340,8 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { } serverID.With(prometheus.Labels{"server_id": b.cluster.nodeID.String()}).Set(1) srv.cluster.SetVersionChangedNotifier(srv.clusterVersionChanged) + //TODO To be removed when RaftCluster does not need the v2store + srv.cluster.SetStore(srv.v2store) srv.applyV2 = NewApplierV2(cfg.Logger, srv.v2store, srv.cluster) srv.be = b.storage.backend.be @@ -1043,17 +1045,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) { lg.Info("restored auth store") } - lg.Info("restoring v2 store") - if err := s.v2store.Recovery(toApply.snapshot.Data); err != nil { - lg.Panic("failed to restore v2 store", zap.Error(err)) + if err := serverstorage.AssertV2DeprecationStage(lg, s.Cfg.V2Deprecation); err != nil { + lg.Panic("illegal v2store deprecation stage. This version does not recover from v2 snapshot", zap.Error(err)) } - if err := serverstorage.AssertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil { - lg.Panic("illegal v2store content", zap.Error(err)) - } - - lg.Info("restored v2 store") - s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe)) lg.Info("restoring cluster configuration") diff --git a/server/storage/util.go b/server/storage/util.go index e1996cfe581..af7f00d450c 100644 --- a/server/storage/util.go +++ b/server/storage/util.go @@ -25,24 +25,14 @@ import ( "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" - "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/raft/v3/raftpb" ) -// AssertNoV2StoreContent -> depending on the deprecation stage, warns or report an error -// if the v2store contains custom content. -func AssertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error { - metaOnly, err := membership.IsMetaStoreOnly(st) - if err != nil { - return err +func AssertV2DeprecationStage(lg *zap.Logger, deprecationStage config.V2DeprecationEnum) error { + //supported stages are "write-only-drop-data" and "gone" + if !deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY_DROP) { + return fmt.Errorf("Unsupported stage --v2-deprecation=%s", deprecationStage) } - if metaOnly { - return nil - } - if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) { - return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage) - } - lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.") return nil }