From 0795f6b9571a34ea11681c82a622cb76560c7f4d Mon Sep 17 00:00:00 2001 From: Jiayin Mao Date: Mon, 27 Jan 2025 14:02:00 +0000 Subject: [PATCH] migrate experimental-memory-mlock flag to memory-mlock Signed-off-by: Jiayin Mao --- server/config/config.go | 4 +- server/embed/config.go | 18 +++++++-- server/embed/etcd.go | 2 +- server/etcdmain/config.go | 5 +++ server/etcdmain/config_test.go | 73 ++++++++++++++++++++++++++++++++++ server/etcdmain/help.go | 4 +- server/storage/backend.go | 2 +- 7 files changed, 99 insertions(+), 9 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 93bfb3362e3e..0eb476d19451 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -184,13 +184,13 @@ type ServerConfig struct { DowngradeCheckTime time.Duration - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. - ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` + MemoryMlock bool `json:"memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use // a shared buffer in its readonly check operations. diff --git a/server/embed/config.go b/server/embed/config.go index 618457f8513e..4f95956f3388 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -132,7 +132,7 @@ var ( // in 3.6, we are migration all the --experimental flags to feature gate and flags without the prefix. // This is the mapping from the non boolean `experimental-` to the new flags. // TODO: delete in v3.7 - experimentalNonBoolFlagMigrationMap = map[string]string{ + experimentalFlagMigrationMap = map[string]string{ "experimental-compact-hash-check-time": "compact-hash-check-time", "experimental-corrupt-check-time": "corrupt-check-time", "experimental-compaction-batch-limit": "compaction-batch-limit", @@ -140,6 +140,7 @@ var ( "experimental-warning-apply-duration": "warning-apply-duration", "experimental-bootstrap-defrag-threshold-megabytes": "bootstrap-defrag-threshold-megabytes", "experimental-max-learners": "max-learners", + "experimental-memory-mlock": "memory-mlock", } ) @@ -488,12 +489,17 @@ type Config struct { ExperimentalDowngradeCheckTime time.Duration `json:"experimental-downgrade-check-time"` - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. + MemoryMlock bool `json:"memory-mlock"` + + // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use MemoryMlock instead. + // TODO: Delete in v3.7 ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. @@ -610,7 +616,9 @@ func NewConfig() *Config { LogRotationConfigJSON: DefaultLogRotationConfig, EnableGRPCGateway: true, - ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + MemoryMlock: false, + // TODO: delete in v3.7 ExperimentalMemoryMlock: false, ExperimentalStopGRPCServiceOnDefrag: false, MaxLearners: membership.DefaultMaxLearners, @@ -825,7 +833,9 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.DurationVar(&cfg.WarningApplyDuration, "warning-apply-duration", cfg.WarningApplyDuration, "Time duration after which a warning is generated if watch progress takes more time.") fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.") fs.DurationVar(&cfg.ExperimentalWarningUnaryRequestDuration, "experimental-warning-unary-request-duration", cfg.ExperimentalWarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time. It's deprecated, and will be decommissioned in v3.7. Use --warning-unary-request-duration instead.") + // TODO: delete in v3.7 fs.BoolVar(&cfg.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") + fs.BoolVar(&cfg.MemoryMlock, "memory-mlock", cfg.MemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") fs.BoolVar(&cfg.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") // TODO: delete in v3.7 @@ -1047,7 +1057,7 @@ func updateMinMaxVersions(info *transport.TLSInfo, min, max string) { func (cfg *Config) Validate() error { // make sure there is no conflict in the flag settings in the ExperimentalNonBoolFlagMigrationMap // TODO: delete in v3.7 - for oldFlag, newFlag := range experimentalNonBoolFlagMigrationMap { + for oldFlag, newFlag := range experimentalFlagMigrationMap { if cfg.FlagsExplicitlySet[oldFlag] && cfg.FlagsExplicitlySet[newFlag] { return fmt.Errorf("cannot set --%s and --%s at the same time, please use --%s only", oldFlag, newFlag, newFlag) } diff --git a/server/embed/etcd.go b/server/embed/etcd.go index bc9725171a66..88c476909907 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -233,7 +233,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { DowngradeCheckTime: cfg.ExperimentalDowngradeCheckTime, WarningApplyDuration: cfg.WarningApplyDuration, WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, - ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, + MemoryMlock: cfg.MemoryMlock, BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, MaxLearners: cfg.MaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 907bf6bfed15..bfe3ad9075eb 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -71,6 +71,7 @@ var ( "experimental-warning-apply-duration": "--experimental-warning-apply-duration is deprecated in v3.6 and will be decommissioned in v3.7. Use '--warning-apply-duration' instead.", "experimental-bootstrap-defrag-threshold-megabytes": "--experimental-bootstrap-defrag-threshold-megabytes is deprecated in v3.6 and will be decommissioned in v3.7. Use '--bootstrap-defrag-threshold-megabytes' instead.", "experimental-max-learners": "--experimental-max-learners is deprecated in v3.6 and will be decommissioned in v3.7. Use '--max-learners' instead.", + "experimental-memory-mlock": "--experimental-memory-mlock is deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead.", } ) @@ -204,6 +205,10 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.MaxLearners = cfg.ec.ExperimentalMaxLearners } + if cfg.ec.FlagsExplicitlySet["experimental-memory-mlock"] { + cfg.ec.MemoryMlock = cfg.ec.ExperimentalMemoryMlock + } + // `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. cfg.ec.V2Deprecation = cconfig.V2DeprDefault diff --git a/server/etcdmain/config_test.go b/server/etcdmain/config_test.go index c8380d9409d2..5c4ef120abf8 100644 --- a/server/etcdmain/config_test.go +++ b/server/etcdmain/config_test.go @@ -946,6 +946,79 @@ func TestMaxLearnersFlagMigration(t *testing.T) { } } +// TestMemoryMlockFlagMigration tests the migration from +// --experimental-memory-mlock to --memory-mlock +// TODO: delete in v3.7 +func TestMemoryMlockFlagMigration(t *testing.T) { + testCases := []struct { + name string + memoryMlock bool + experimentalMemoryMlock bool + expectedMemoryMlock bool + expectErr bool + }{ + { + name: "default", + expectedMemoryMlock: false, + }, + { + name: "cannot set both experimental flag and non experimental flag", + memoryMlock: true, + experimentalMemoryMlock: true, + expectErr: true, + }, + { + name: "can set experimental flag", + experimentalMemoryMlock: true, + expectedMemoryMlock: true, + }, + { + name: "can set non experimental flag", + memoryMlock: true, + expectedMemoryMlock: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmdLineArgs := []string{} + yc := struct { + MemoryMlock bool `json:"memory-mlock,omitempty"` + ExperimentalMemoryMlock bool `json:"experimental-memory-mlock,omitempty"` + }{} + + if tc.memoryMlock { + cmdLineArgs = append(cmdLineArgs, "--memory-mlock") + yc.MemoryMlock = tc.memoryMlock + } + + if tc.experimentalMemoryMlock { + cmdLineArgs = append(cmdLineArgs, "--experimental-memory-mlock") + yc.ExperimentalMemoryMlock = tc.experimentalMemoryMlock + } + + cfgFromCmdLine, errFromCmdLine, cfgFromFile, errFromFile := generateCfgsFromFileAndCmdLine(t, yc, cmdLineArgs) + + if tc.expectErr { + if errFromCmdLine == nil || errFromFile == nil { + t.Fatal("expect parse error") + } + return + } + + if errFromCmdLine != nil || errFromFile != nil { + t.Fatal("error parsing config") + } + + if cfgFromCmdLine.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromCmdLine.ec.MemoryMlock) + } + if cfgFromFile.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromFile.ec.MemoryMlock) + } + }) + } +} + // TODO delete in v3.7 func generateCfgsFromFileAndCmdLine(t *testing.T, yc any, cmdLineArgs []string) (*config, error, *config, error) { b, err := yaml.Marshal(&yc) diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 986c0d5198f8..2273d4989423 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -75,6 +75,8 @@ Member: Maximum number of snapshot files to retain (0 is unlimited). Deprecated in v3.6 and will be decommissioned in v3.7. --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `' Maximum number of wal files to retain (0 is unlimited). + --memory-mlock + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --quota-backend-bytes '0' Raise alarms when backend size exceeds the given quota (0 defaults to low space quota). --backend-bbolt-freelist-type 'map' @@ -322,7 +324,7 @@ Experimental feature: --experimental-enable-lease-checkpoint-persist 'false' Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled. --experimental-memory-mlock - Enable to enforce etcd pages (in particular bbolt) to stay in RAM. + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. Deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead. --experimental-snapshot-catchup-entries Number of entries for a slow follower to catch up after compacting the raft storage entries. --experimental-stop-grpc-service-on-defrag diff --git a/server/storage/backend.go b/server/storage/backend.go index b7b0d6861ad5..7db61f9fae56 100644 --- a/server/storage/backend.go +++ b/server/storage/backend.go @@ -50,7 +50,7 @@ func newBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { // permit 10% excess over quota for disarm bcfg.MmapSize = uint64(cfg.QuotaBackendBytes + cfg.QuotaBackendBytes/10) } - bcfg.Mlock = cfg.ExperimentalMemoryMlock + bcfg.Mlock = cfg.MemoryMlock bcfg.Hooks = hooks return backend.New(bcfg) }