etcdserver: Implement running defrag if freeable space will exceed pr…

…ivided threshold

server/config/config.go

@@ -188,6 +188,10 @@ type ServerConfig struct {
 	// ExperimentalTxnModeWriteWithSharedBuffer enable write transaction to use
 	// a shared buffer in its readonly check operations.
 	ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"`
+
+	// ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to
+	// consider running defrag during bootstrap. Needs to be set to non-zero value to take effect.
+	ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/embed/config.go

@@ -314,6 +314,9 @@ type Config struct {
 	// ExperimentalWarningApplyDuration is the time duration after which a warning is generated if applying request
 	// takes more time than this value.
 	ExperimentalWarningApplyDuration time.Duration `json:"experimental-warning-apply-duration"`
+	// ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to
+	// consider running defrag during bootstrap. Needs to be set to non-zero value to take effect.
+	ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"`
 
 	// ForceNewCluster starts a new cluster even if previously started; unsafe.
 	ForceNewCluster bool `json:"force-new-cluster"`

server/embed/etcd.go

@@ -225,6 +225,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		WarningApplyDuration:                     cfg.ExperimentalWarningApplyDuration,
 		ExperimentalMemoryMlock:                  cfg.ExperimentalMemoryMlock,
 		ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer,
+		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {

server/etcdmain/config.go

@@ -276,6 +276,7 @@ func newConfig() *config {
 	fs.DurationVar(&cfg.ec.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ec.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.")
 	fs.BoolVar(&cfg.ec.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ec.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.")
 	fs.BoolVar(&cfg.ec.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.")
+	fs.UintVar(&cfg.ec.ExperimentalBootstrapDefragThresholdMegabytes, "experimental-bootstrap-defrag-threshold-megabytes", 0, "Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect.")
 
 	// unsafe
 	fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.")

server/etcdmain/help.go

@@ -236,6 +236,8 @@ Experimental feature:
 	Warning is generated if requests take more than this duration.
   --experimental-txn-mode-write-with-shared-buffer 'true'
     Enable the write transaction to use a shared buffer in its readonly check operations.
+  --experimental-bootstrap-defrag-threshold-megabytes
+    Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect.
 
 Unsafe feature:
   --force-new-cluster 'false'

server/etcdserver/server.go

@@ -362,6 +362,13 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	ci.SetBackend(be)
 	cindex.CreateMetaBucket(be.BatchTx())
 
+	if cfg.ExperimentalBootstrapDefragThresholdMegabytes != 0 {
+		err := maybeDefragBackend(cfg, be)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	defer func() {
 		if err != nil {
 			be.Close()
@@ -2580,3 +2587,22 @@ func (s *EtcdServer) IsMemberExist(id types.ID) bool {
 func (s *EtcdServer) raftStatus() raft.Status {
 	return s.r.Node.Status()
 }
+
+func maybeDefragBackend(cfg config.ServerConfig, be backend.Backend) error {
+	size := be.Size()
+	sizeInUse := be.SizeInUse()
+	freeableMemory := uint(size - sizeInUse)
+	thresholdBytes := cfg.ExperimentalBootstrapDefragThresholdMegabytes * 1024 * 1024
+	if freeableMemory < thresholdBytes {
+		cfg.Logger.Info("Skipping defragmentation",
+			zap.Int64("current-db-size-bytes", size),
+			zap.String("current-db-size", humanize.Bytes(uint64(size))),
+			zap.Int64("current-db-size-in-use-bytes", sizeInUse),
+			zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse))),
+			zap.Uint("experimental-bootstrap-defrag-threshold-bytes", thresholdBytes),
+			zap.String("experimental-bootstrap-defrag-threshold", humanize.Bytes(uint64(thresholdBytes))),
+		)
+		return nil
+	}
+	return be.Defrag()
+}

tests/e2e/etcd_config_test.go

@@ -319,3 +319,18 @@ func TestGrpcproxyAndCommonName(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestBootstrapDefragFlag(t *testing.T) {
+	skipInShortMode(t)
+
+	proc, err := spawnCmd([]string{binDir + "/etcd", "--experimental-bootstrap-defrag-threshold-megabytes", "1000"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = waitReadyExpectProc(proc, []string{"Skipping defragmentation"}); err != nil {
+		t.Fatal(err)
+	}
+	if err = proc.Stop(); err != nil {
+		t.Fatal(err)
+	}
+}