From 40c5a6fc1163233aed78e073081956870ad00ce1 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Fri, 10 Jun 2022 14:33:49 +0800 Subject: [PATCH] Add a new flag --experimental-enable-lease-v2-renew to enable or disable the legacy renew The legacy renew will be deprecated in next release (3.7), but in order to to backward & forward compatible, so we add this new flag. It's enabled by default in 3.6, so that we continue to use the legacy renew implementation. --- server/config/config.go | 6 ++++++ server/embed/config.go | 4 +++- server/embed/etcd.go | 5 +++++ server/etcdmain/config.go | 1 + server/etcdmain/help.go | 2 ++ server/etcdserver/api/v3rpc/lease.go | 1 - server/etcdserver/v3_server.go | 15 ++++++++++++++- 7 files changed, 31 insertions(+), 3 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 75d7df6c4a68..c22d55bd6ddd 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -164,6 +164,12 @@ type ServerConfig struct { // LeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. LeaseCheckpointPersist bool + // ExperimentalEnableLeaseV2Renew enables the legacy renew implementation. + // When it's enabled, if a follower receives the renew request from client, + // it just forward the request to the leader. + // Otherwise, each member just submits the request to raft. + ExperimentalEnableLeaseV2Renew bool + EnableGRPCGateway bool // ExperimentalEnableDistributedTracing enables distributed tracing using OpenTelemetry protocol. diff --git a/server/embed/config.go b/server/embed/config.go index 2cacf5ea4f6d..feae07fee459 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -323,7 +323,9 @@ type Config struct { // Deprecated in v3.6. // TODO: Delete in v3.7 ExperimentalEnableLeaseCheckpointPersist bool `json:"experimental-enable-lease-checkpoint-persist"` - ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` + ExperimentalEnableLeaseV2Renew bool `json:"experimental-enable-lease-v2-renew"` + + ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` // ExperimentalCompactionSleepInterval is the sleep interval between every etcd compaction loop. ExperimentalCompactionSleepInterval time.Duration `json:"experimental-compaction-sleep-interval"` ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval"` diff --git a/server/embed/etcd.go b/server/embed/etcd.go index ffd239c79985..5e2ec57945b8 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -209,6 +209,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { UnsafeNoFsync: cfg.UnsafeNoFsync, EnableLeaseCheckpoint: cfg.ExperimentalEnableLeaseCheckpoint, LeaseCheckpointPersist: cfg.ExperimentalEnableLeaseCheckpointPersist, + ExperimentalEnableLeaseV2Renew: cfg.ExperimentalEnableLeaseV2Renew, CompactionBatchLimit: cfg.ExperimentalCompactionBatchLimit, CompactionSleepInterval: cfg.ExperimentalCompactionSleepInterval, WatchProgressNotifyInterval: cfg.ExperimentalWatchProgressNotifyInterval, @@ -346,6 +347,10 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized zap.String("discovery-url", sc.DiscoveryURL), zap.String("discovery-proxy", sc.DiscoveryProxy), + zap.Bool("experimental-enable-lease-checkpoint", sc.EnableLeaseCheckpoint), + zap.Bool("experimental-enable-lease-checkpoint-persist", sc.LeaseCheckpointPersist), + zap.Bool("experimental-enable-lease-v2-renew", sc.ExperimentalEnableLeaseV2Renew), + zap.String("discovery-token", sc.DiscoveryCfg.Token), zap.String("discovery-endpoints", strings.Join(sc.DiscoveryCfg.Endpoints, ",")), zap.String("discovery-dial-timeout", sc.DiscoveryCfg.DialTimeout.String()), diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 8091589dfcea..2587fc189c9e 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -259,6 +259,7 @@ func newConfig() *config { fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.") fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change.") + fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseV2Renew, "experimental-enable-lease-v2-renew", true, "Enable lease v2 renew implementation. Will be deprecated in 3.7.") // TODO: delete in v3.7 fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpointPersist, "experimental-enable-lease-checkpoint-persist", false, "Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled.") fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.") diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index efa2a77e8e08..11f5a2a9ad39 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -246,6 +246,8 @@ Experimental feature: Duration of time between cluster corruption check passes. --experimental-enable-lease-checkpoint 'false' ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. + --experimental-enable-lease-v2-renew 'true' + ExperimentalEnableLeaseV2Renew enables the legacy lease v2 renew implementation. Will be deprecated in 3.7. --experimental-compaction-batch-limit 1000 ExperimentalCompactionBatchLimit sets the maximum revisions deleted in each compaction batch. --experimental-peer-skip-client-san-verification 'false' diff --git a/server/etcdserver/api/v3rpc/lease.go b/server/etcdserver/api/v3rpc/lease.go index 75f45a474df3..9eda3ed14e74 100644 --- a/server/etcdserver/api/v3rpc/lease.go +++ b/server/etcdserver/api/v3rpc/lease.go @@ -129,7 +129,6 @@ func (ls *LeaseServer) leaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) erro // or remote leader. // Without this, a lease might be revoked at rev 3 but client can see the keepalive succeeded // at rev 4. - // todo(ahrtr): remove respForLeaseNotFound, we don't need to process ErrLeaseNotFound separately. respForLeaseNotFound := &pb.LeaseKeepAliveResponse{ID: req.ID, Header: &pb.ResponseHeader{}} ls.hdr.fill(respForLeaseNotFound.Header) diff --git a/server/etcdserver/v3_server.go b/server/etcdserver/v3_server.go index 1ec1f52db3bf..6688d51e6da9 100644 --- a/server/etcdserver/v3_server.go +++ b/server/etcdserver/v3_server.go @@ -275,6 +275,19 @@ func (s *EtcdServer) LeaseRevoke(ctx context.Context, r *pb.LeaseRevokeRequest) return resp.(*pb.LeaseRevokeResponse), nil } +func (s *EtcdServer) LeaseRenew(ctx context.Context, r *pb.LeaseKeepAliveRequest) (*pb.LeaseKeepAliveResponse, error) { + if s.Cfg.ExperimentalEnableLeaseV2Renew { + resp := &pb.LeaseKeepAliveResponse{ID: r.ID, Header: s.newHeader()} + + var err error + resp.TTL, err = s.leaseRenewV2(ctx, lease.LeaseID(r.ID)) + + return resp, err + } + + return s.leaseRenewV3(ctx, r) +} + func (s *EtcdServer) leaseRenewV2(ctx context.Context, id lease.LeaseID) (int64, error) { if s.isLeader() { if err := s.waitAppliedIndex(); err != nil { @@ -316,7 +329,7 @@ func (s *EtcdServer) leaseRenewV2(ctx context.Context, id lease.LeaseID) (int64, return -1, errors.ErrCanceled } -func (s *EtcdServer) LeaseRenew(ctx context.Context, r *pb.LeaseKeepAliveRequest) (*pb.LeaseKeepAliveResponse, error) { +func (s *EtcdServer) leaseRenewV3(ctx context.Context, r *pb.LeaseKeepAliveRequest) (*pb.LeaseKeepAliveResponse, error) { resp, err := s.raftRequestOnce(ctx, pb.InternalRaftRequest{LeaseRenew: r}) if err != nil { return nil, err