From b8b764c229ff64bf695cff4bf3133c53c3a27534 Mon Sep 17 00:00:00 2001 From: Jiayin Mao Date: Sat, 25 Jan 2025 22:12:22 +0000 Subject: [PATCH] storage: promote experimental-memory-mlock flag. Since the memory mlock feature has been added as an experimental feature since v3.5 we want to promote it as a stable feature as memory-mlock in v3.6. In v3.7 we will retire the experimental-memory-mlock flag. Signed-off-by: Jiayin Mao --- server/config/config.go | 5 ++++- server/embed/config.go | 12 ++++++++++-- server/embed/etcd.go | 1 + server/etcdmain/help.go | 2 ++ server/storage/backend.go | 3 ++- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index dee41b86de5..d6aca50b1db 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -184,12 +184,15 @@ type ServerConfig struct { DowngradeCheckTime time.Duration - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. + MemoryMlock bool `json:"memory-mlock"` + // ExperimentalMemoryMlock has the same effect as MemoryMlock + // TODO: delete in v3.7 ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use diff --git a/server/embed/config.go b/server/embed/config.go index a66adf97c2b..2976782a422 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -471,12 +471,17 @@ type Config struct { ExperimentalDowngradeCheckTime time.Duration `json:"experimental-downgrade-check-time"` - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. + MemoryMlock bool `json:"memory-mlock"` + + // ExperimentalMemoryMlock has the same effect as MemoryMlock in v3.6. + // Deprecated in v3.6 and will be decommissioned in v3.7. + // TODO: Delete in v3.7 ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. @@ -582,7 +587,8 @@ func NewConfig() *Config { BcryptCost: uint(bcrypt.DefaultCost), AuthTokenTTL: 300, - PreVote: true, + PreVote: true, + MemoryMlock: false, loggerMu: new(sync.RWMutex), logger: nil, @@ -802,6 +808,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.DurationVar(&cfg.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.") fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.") fs.DurationVar(&cfg.ExperimentalWarningUnaryRequestDuration, "experimental-warning-unary-request-duration", cfg.ExperimentalWarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time. It's deprecated, and will be decommissioned in v3.7. Use --warning-unary-request-duration instead.") + fs.BoolVar(&cfg.MemoryMlock, "memory-mlock", cfg.MemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") + // TODO: delete in v3.7 fs.BoolVar(&cfg.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") fs.BoolVar(&cfg.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") diff --git a/server/embed/etcd.go b/server/embed/etcd.go index f5fa2c2c692..1877db77831 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -208,6 +208,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { CorruptCheckTime: cfg.CorruptCheckTime, CompactHashCheckTime: cfg.CompactHashCheckTime, PreVote: cfg.PreVote, + MemoryMlock: cfg.MemoryMlock, Logger: cfg.logger, ForceNewCluster: cfg.ForceNewCluster, EnableGRPCGateway: cfg.EnableGRPCGateway, diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 832716fb24f..f6e1fa78516 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -107,6 +107,8 @@ Member: Write timeout set on each rafthttp connection --feature-gates '' A set of key=value pairs that describe server level feature gates for alpha/experimental features. Options are:` + "\n " + strings.Join(features.NewDefaultServerFeatureGate("", nil).KnownFeatures(), "\n ") + ` + --memory-mlock 'false' + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. Clustering: --initial-advertise-peer-urls 'http://localhost:2380' diff --git a/server/storage/backend.go b/server/storage/backend.go index b7b0d6861ad..51c01e4240d 100644 --- a/server/storage/backend.go +++ b/server/storage/backend.go @@ -50,7 +50,8 @@ func newBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { // permit 10% excess over quota for disarm bcfg.MmapSize = uint64(cfg.QuotaBackendBytes + cfg.QuotaBackendBytes/10) } - bcfg.Mlock = cfg.ExperimentalMemoryMlock + // TODO: remove the cfg.ExperimentalMemoryMlock part in v3.7 + bcfg.Mlock = cfg.MemoryMlock || cfg.ExperimentalMemoryMlock bcfg.Hooks = hooks return backend.New(bcfg) }