From f5c17babe658f97524ac7d00485af2d4a39fa78f Mon Sep 17 00:00:00 2001
From: Bogdan Kanivets <bkanivets@apple.com>
Date: Thu, 1 Jun 2023 01:15:41 -0700
Subject: [PATCH] allow downgrade from 3.5

Signed-off-by: Siyuan Zhang <sizhang@google.com>
---
 etcdserver/api/capability.go         |  1 +
 etcdserver/api/membership/cluster.go |  7 ++++++-
 mvcc/kvstore.go                      | 21 +++++++++++++++++++++
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/etcdserver/api/capability.go b/etcdserver/api/capability.go
index 8b13f4742950..09b754d1343b 100644
--- a/etcdserver/api/capability.go
+++ b/etcdserver/api/capability.go
@@ -41,6 +41,7 @@ var (
 		"3.2.0": {AuthCapability: true, V3rpcCapability: true},
 		"3.3.0": {AuthCapability: true, V3rpcCapability: true},
 		"3.4.0": {AuthCapability: true, V3rpcCapability: true},
+		"3.5.0": {AuthCapability: true, V3rpcCapability: true},
 	}
 
 	enableMapMu sync.RWMutex
diff --git a/etcdserver/api/membership/cluster.go b/etcdserver/api/membership/cluster.go
index ae1a56080059..a9e2f4bd3219 100644
--- a/etcdserver/api/membership/cluster.go
+++ b/etcdserver/api/membership/cluster.go
@@ -787,10 +787,15 @@ func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *R
 }
 
 func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version) {
+	if cv == nil {
+		return
+	}
 	lv := semver.Must(semver.NewVersion(version.Version))
 	// only keep major.minor version for comparison against cluster version
 	lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
-	if cv != nil && lv.LessThan(*cv) {
+	// allow 3.4 server to join 3.5 cluster version, since the data is compatible after `etcdutil migrate` the data_dir.
+	allowedDowngradeVersion := &semver.Version{Major: cv.Major, Minor: cv.Minor - 1}
+	if lv.LessThan(*cv) && !lv.Equal(*allowedDowngradeVersion) {
 		if lg != nil {
 			lg.Fatal(
 				"invalid downgrade; server version is lower than determined cluster version",
diff --git a/mvcc/kvstore.go b/mvcc/kvstore.go
index 6dc4f30bc0ce..c4e7353c9d06 100644
--- a/mvcc/kvstore.go
+++ b/mvcc/kvstore.go
@@ -379,6 +379,27 @@ func (s *store) restore() error {
 	tx := s.b.BatchTx()
 	tx.Lock()
 
+	// new keys introduced in 3.5.
+	invalidMetaBucketKeys := map[string]struct{}{
+		"term":      {},
+		"confState": {},
+	}
+	err := tx.UnsafeForEach(metaBucketName, func(k []byte, v []byte) error {
+		kstr := string(k)
+		if _, found := invalidMetaBucketKeys[kstr]; found {
+			return fmt.Errorf("invalid key: %s found in 3.4 meta bucket. use `etcdutl migrate` to migrate the data_dir before downgrade!", kstr)
+		}
+		return nil
+	})
+	if err != nil {
+		if s.lg != nil {
+			s.lg.Panic("failed to verify meta bucket schema",
+				zap.Error(err))
+		} else {
+			plog.Fatal(err)
+		}
+	}
+
 	_, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0)
 	if len(finishedCompactBytes) != 0 {
 		s.revMu.Lock()