diff --git a/server/config/config.go b/server/config/config.go index fc3c72b756a..7776cf049a6 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -184,13 +184,13 @@ type ServerConfig struct { DowngradeCheckTime time.Duration - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. - ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` + MemoryMlock bool `json:"memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use // a shared buffer in its readonly check operations. diff --git a/server/embed/config.go b/server/embed/config.go index 890a22e8c7d..881a6521ba6 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -484,12 +484,17 @@ type Config struct { ExperimentalDowngradeCheckTime time.Duration `json:"experimental-downgrade-check-time"` - // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. + MemoryMlock bool `json:"memory-mlock"` + + // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // Deprecated in v3.6 and will be decommissioned in v3.7. Use MemoryMlock instead. + // TODO: Delete in v3.7 ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. @@ -606,7 +611,9 @@ func NewConfig() *Config { LogRotationConfigJSON: DefaultLogRotationConfig, EnableGRPCGateway: true, - ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + MemoryMlock: false, + // TODO: delete in v3.7 ExperimentalMemoryMlock: false, ExperimentalStopGRPCServiceOnDefrag: false, ExperimentalMaxLearners: membership.DefaultMaxLearners, @@ -819,7 +826,9 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.DurationVar(&cfg.WarningApplyDuration, "warning-apply-duration", cfg.WarningApplyDuration, "Time duration after which a warning is generated if watch progress takes more time.") fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.") fs.DurationVar(&cfg.ExperimentalWarningUnaryRequestDuration, "experimental-warning-unary-request-duration", cfg.ExperimentalWarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time. It's deprecated, and will be decommissioned in v3.7. Use --warning-unary-request-duration instead.") + // TODO: delete in v3.7 fs.BoolVar(&cfg.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") + fs.BoolVar(&cfg.MemoryMlock, "memory-mlock", cfg.MemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") fs.BoolVar(&cfg.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") // TODO: delete in v3.7 diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 41ca367aa2d..c4ae0fa6bfd 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -233,7 +233,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { DowngradeCheckTime: cfg.ExperimentalDowngradeCheckTime, WarningApplyDuration: cfg.WarningApplyDuration, WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, - ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, + MemoryMlock: cfg.MemoryMlock, BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, ExperimentalMaxLearners: cfg.ExperimentalMaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index b5beade8d29..6a2d4d7baa9 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -70,6 +70,7 @@ var ( "experimental-watch-progress-notify-interval": "--experimental-watch-progress-notify-interval is deprecated in v3.6 and will be decommissioned in v3.7. Use '--watch-progress-notify-interval' instead.", "experimental-warning-apply-duration": "--experimental-warning-apply-duration is deprecated in v3.6 and will be decommissioned in v3.7. Use '--warning-apply-duration' instead.", "experimental-bootstrap-defrag-threshold-megabytes": "--experimental-bootstrap-defrag-threshold-megabytes is deprecated in v3.6 and will be decommissioned in v3.7. Use '--bootstrap-defrag-threshold-megabytes' instead.", + "experimental-memory-mlock": "--experimental-memory-mlock is deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead.", } ) @@ -199,6 +200,10 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.BootstrapDefragThresholdMegabytes = cfg.ec.ExperimentalBootstrapDefragThresholdMegabytes } + if cfg.ec.FlagsExplicitlySet["experimental-memory-mlock"] { + cfg.ec.MemoryMlock = cfg.ec.ExperimentalMemoryMlock + } + // `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. cfg.ec.V2Deprecation = cconfig.V2DeprDefault diff --git a/server/etcdmain/config_test.go b/server/etcdmain/config_test.go index 15d724b2228..3a75af5664f 100644 --- a/server/etcdmain/config_test.go +++ b/server/etcdmain/config_test.go @@ -882,6 +882,65 @@ func TestBootstrapDefragThresholdMegabytesFlagMigration(t *testing.T) { } } +// TestMemoryMlockFlagMigration tests the migration from +// --experimental-memory-mlock to --memory-mlock +// TODO: delete in v3.7 +func TestMemoryMlockFlagMigration(t *testing.T) { + testCases := []struct { + name string + memoryMlock bool + experimentalMemoryMlock bool + expectedMemoryMlock bool + }{ + { + name: "default", + expectedMemoryMlock: false, + }, + { + name: "can set experimental flag", + experimentalMemoryMlock: true, + expectedMemoryMlock: true, + }, + { + name: "can set non experimental flag", + memoryMlock: true, + expectedMemoryMlock: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmdLineArgs := []string{} + yc := struct { + MemoryMlock bool `json:"memory-mlock,omitempty"` + ExperimentalMemoryMlock bool `json:"experimental-memory-mlock,omitempty"` + }{} + + if tc.memoryMlock { + cmdLineArgs = append(cmdLineArgs, "--memory-mlock") + yc.MemoryMlock = tc.memoryMlock + } + + if tc.experimentalMemoryMlock { + cmdLineArgs = append(cmdLineArgs, "--experimental-memory-mlock") + yc.ExperimentalMemoryMlock = tc.experimentalMemoryMlock + } + + cfgFromCmdLine, errFromCmdLine, cfgFromFile, errFromFile := generateCfgsFromFileAndCmdLine(t, yc, cmdLineArgs) + + if errFromCmdLine != nil || errFromFile != nil { + t.Fatal("error parsing config") + } + + if cfgFromCmdLine.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromCmdLine.ec.MemoryMlock) + } + if cfgFromFile.ec.MemoryMlock != tc.expectedMemoryMlock { + t.Errorf("expected MemoryMlock=%v, got %v", tc.expectedMemoryMlock, cfgFromFile.ec.MemoryMlock) + } + }) + } +} + // TODO delete in v3.7 func generateCfgsFromFileAndCmdLine(t *testing.T, yc any, cmdLineArgs []string) (*config, error, *config, error) { b, err := yaml.Marshal(&yc) diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 9d48fe0c29d..a9f1ba7fe41 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -75,6 +75,8 @@ Member: Maximum number of snapshot files to retain (0 is unlimited). Deprecated in v3.6 and will be decommissioned in v3.7. --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `' Maximum number of wal files to retain (0 is unlimited). + --memory-mlock + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --quota-backend-bytes '0' Raise alarms when backend size exceeds the given quota (0 defaults to low space quota). --backend-bbolt-freelist-type 'map' @@ -320,7 +322,7 @@ Experimental feature: --experimental-enable-lease-checkpoint-persist 'false' Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled. --experimental-memory-mlock - Enable to enforce etcd pages (in particular bbolt) to stay in RAM. + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. Deprecated in v3.6 and will be decommissioned in v3.7. Use '--memory-mlock' instead. --experimental-snapshot-catchup-entries Number of entries for a slow follower to catch up after compacting the raft storage entries. --experimental-stop-grpc-service-on-defrag diff --git a/server/storage/backend.go b/server/storage/backend.go index b7b0d6861ad..7db61f9fae5 100644 --- a/server/storage/backend.go +++ b/server/storage/backend.go @@ -50,7 +50,7 @@ func newBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { // permit 10% excess over quota for disarm bcfg.MmapSize = uint64(cfg.QuotaBackendBytes + cfg.QuotaBackendBytes/10) } - bcfg.Mlock = cfg.ExperimentalMemoryMlock + bcfg.Mlock = cfg.MemoryMlock bcfg.Hooks = hooks return backend.New(bcfg) }