From 8d5be6ed77cf5cdc521e1118c8ba22d6ab219735 Mon Sep 17 00:00:00 2001 From: Jiayin Mao Date: Mon, 27 Jan 2025 14:02:00 +0000 Subject: [PATCH] migrate experimental-memory-mlock flag to memory-mlock Signed-off-by: Jiayin Mao --- server/config/config.go | 4 +-- server/embed/config.go | 13 ++++++-- server/embed/etcd.go | 2 +- server/etcdmain/config.go | 5 +++ server/etcdmain/config_test.go | 59 ++++++++++++++++++++++++++++++++++ server/etcdmain/help.go | 4 ++- server/storage/backend.go | 2 +- 7 files changed, 82 insertions(+), 7 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 93bfb3362e3..0eb476d1945 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -184,13 +184,13 @@ type ServerConfig struct { DowngradeCheckTime time.Duration - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. - ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` + MemoryMlock bool `json:"memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use // a shared buffer in its readonly check operations. diff --git a/server/embed/config.go b/server/embed/config.go index 618457f8513..614f98c512f 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -488,12 +488,17 @@ type Config struct { ExperimentalDowngradeCheckTime time.Duration `json:"experimental-downgrade-check-time"` - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. + MemoryMlock bool `json:"memory-mlock"` + + // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use MemoryMlock instead. + // TODO: Delete in v3.7 ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. @@ -610,7 +615,9 @@ func NewConfig() *Config { LogRotationConfigJSON: DefaultLogRotationConfig, EnableGRPCGateway: true, - ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + MemoryMlock: false, + // TODO: delete in v3.7 ExperimentalMemoryMlock: false, ExperimentalStopGRPCServiceOnDefrag: false, MaxLearners: membership.DefaultMaxLearners, @@ -825,7 +832,9 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.DurationVar(&cfg.WarningApplyDuration, "warning-apply-duration", cfg.WarningApplyDuration, "Time duration after which a warning is generated if watch progress takes more time.") fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.") fs.DurationVar(&cfg.ExperimentalWarningUnaryRequestDuration, "experimental-warning-unary-request-duration", cfg.ExperimentalWarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time. It's deprecated, and will be decommissioned in v3.7. Use --warning-unary-request-duration instead.") + // TODO: delete in v3.7 fs.BoolVar(&cfg.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") + fs.BoolVar(&cfg.MemoryMlock, "memory-mlock", cfg.MemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") fs.BoolVar(&cfg.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") // TODO: delete in v3.7 diff --git a/server/embed/etcd.go b/server/embed/etcd.go index bc9725171a6..88c47690990 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -233,7 +233,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { DowngradeCheckTime: cfg.ExperimentalDowngradeCheckTime, WarningApplyDuration: cfg.WarningApplyDuration, WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, - ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, + MemoryMlock: cfg.MemoryMlock, BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, MaxLearners: cfg.MaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 907bf6bfed1..bfe3ad9075e 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -71,6 +71,7 @@ var ( "experimental-warning-apply-duration": "--experimental-warning-apply-duration is deprecated in v3.6 and will be decommissioned in v3.7. Use '--warning-apply-duration' instead.", "experimental-bootstrap-defrag-threshold-megabytes": "--experimental-bootstrap-defrag-threshold-megabytes is deprecated in v3.6 and will be decommissioned in v3.7. Use '--bootstrap-defrag-threshold-megabytes' instead.", "experimental-max-learners": "--experimental-max-learners is deprecated in v3.6 and will be decommissioned in v3.7. Use '--max-learners' instead.", + "experimental-memory-mlock": "--experimental-memory-mlock is deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead.", } ) @@ -204,6 +205,10 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.MaxLearners = cfg.ec.ExperimentalMaxLearners } + if cfg.ec.FlagsExplicitlySet["experimental-memory-mlock"] { + cfg.ec.MemoryMlock = cfg.ec.ExperimentalMemoryMlock + } + // `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. cfg.ec.V2Deprecation = cconfig.V2DeprDefault diff --git a/server/etcdmain/config_test.go b/server/etcdmain/config_test.go index c8380d9409d..559f023daab 100644 --- a/server/etcdmain/config_test.go +++ b/server/etcdmain/config_test.go @@ -946,6 +946,65 @@ func TestMaxLearnersFlagMigration(t *testing.T) { } } +// TestMemoryMlockFlagMigration tests the migration from +// --experimental-memory-mlock to --memory-mlock +// TODO: delete in v3.7 +func TestMemoryMlockFlagMigration(t *testing.T) { + testCases := []struct { + name string + memoryMlock bool + experimentalMemoryMlock bool + expectedMemoryMlock bool + }{ + { + name: "default", + expectedMemoryMlock: false, + }, + { + name: "can set experimental flag", + experimentalMemoryMlock: true, + expectedMemoryMlock: true, + }, + { + name: "can set non experimental flag", + memoryMlock: true, + expectedMemoryMlock: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmdLineArgs := []string{} + yc := struct { + MemoryMlock bool `json:"memory-mlock,omitempty"` + ExperimentalMemoryMlock bool `json:"experimental-memory-mlock,omitempty"` + }{} + + if tc.memoryMlock { + cmdLineArgs = append(cmdLineArgs, "--memory-mlock") + yc.MemoryMlock = tc.memoryMlock + } + + if tc.experimentalMemoryMlock { + cmdLineArgs = append(cmdLineArgs, "--experimental-memory-mlock") + yc.ExperimentalMemoryMlock = tc.experimentalMemoryMlock + } + + cfgFromCmdLine, errFromCmdLine, cfgFromFile, errFromFile := generateCfgsFromFileAndCmdLine(t, yc, cmdLineArgs) + + if errFromCmdLine != nil || errFromFile != nil { + t.Fatal("error parsing config") + } + + if cfgFromCmdLine.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromCmdLine.ec.MemoryMlock) + } + if cfgFromFile.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromFile.ec.MemoryMlock) + } + }) + } +} + // TODO delete in v3.7 func generateCfgsFromFileAndCmdLine(t *testing.T, yc any, cmdLineArgs []string) (*config, error, *config, error) { b, err := yaml.Marshal(&yc) diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 986c0d5198f..2273d498942 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -75,6 +75,8 @@ Member: Maximum number of snapshot files to retain (0 is unlimited). Deprecated in v3.6 and will be decommissioned in v3.7. --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `' Maximum number of wal files to retain (0 is unlimited). + --memory-mlock + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --quota-backend-bytes '0' Raise alarms when backend size exceeds the given quota (0 defaults to low space quota). --backend-bbolt-freelist-type 'map' @@ -322,7 +324,7 @@ Experimental feature: --experimental-enable-lease-checkpoint-persist 'false' Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled. --experimental-memory-mlock - Enable to enforce etcd pages (in particular bbolt) to stay in RAM. + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. Deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead. --experimental-snapshot-catchup-entries Number of entries for a slow follower to catch up after compacting the raft storage entries. --experimental-stop-grpc-service-on-defrag diff --git a/server/storage/backend.go b/server/storage/backend.go index b7b0d6861ad..7db61f9fae5 100644 --- a/server/storage/backend.go +++ b/server/storage/backend.go @@ -50,7 +50,7 @@ func newBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { // permit 10% excess over quota for disarm bcfg.MmapSize = uint64(cfg.QuotaBackendBytes + cfg.QuotaBackendBytes/10) } - bcfg.Mlock = cfg.ExperimentalMemoryMlock + bcfg.Mlock = cfg.MemoryMlock bcfg.Hooks = hooks return backend.New(bcfg) }