allow downgrade from 3.5

Signed-off-by: Siyuan Zhang <sizhang@google.com>

etcdserver/api/capability.go

@@ -41,6 +41,7 @@ var (
 		"3.2.0": {AuthCapability: true, V3rpcCapability: true},
 		"3.3.0": {AuthCapability: true, V3rpcCapability: true},
 		"3.4.0": {AuthCapability: true, V3rpcCapability: true},
+		"3.5.0": {AuthCapability: true, V3rpcCapability: true},
 	}
 
 	enableMapMu sync.RWMutex

etcdserver/api/membership/cluster.go

@@ -787,10 +787,15 @@ func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *R
 }
 
 func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version) {
+	if cv == nil {
+		return
+	}
 	lv := semver.Must(semver.NewVersion(version.Version))
 	// only keep major.minor version for comparison against cluster version
 	lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
-	if cv != nil && lv.LessThan(*cv) {
+	// allow 3.4 server to join 3.5 cluster version, since the data is compatible after `etcdutil migrate` the data_dir.
+	allowedDowngradeVersion := &semver.Version{Major: cv.Major, Minor: cv.Minor - 1}
+	if lv.LessThan(*cv) && !lv.Equal(*allowedDowngradeVersion) {
 		if lg != nil {
 			lg.Fatal(
 				"invalid downgrade; server version is lower than determined cluster version",

mvcc/kvstore.go

@@ -379,6 +379,27 @@ func (s *store) restore() error {
 	tx := s.b.BatchTx()
 	tx.Lock()
 
+	// new keys introduced in 3.5.
+	invalidMetaBucketKeys := map[string]struct{}{
+		"term":      {},
+		"confState": {},
+	}
+	err := tx.UnsafeForEach(metaBucketName, func(k []byte, v []byte) error {
+		kstr := string(k)
+		if _, found := invalidMetaBucketKeys[kstr]; found {
+			return fmt.Errorf("invalid key: %s found in 3.4 meta bucket. use `etcdutl migrate` to migrate the data_dir before downgrade!", kstr)
+		}
+		return nil
+	})
+	if err != nil {
+		if s.lg != nil {
+			s.lg.Panic("failed to verify meta bucket schema",
+				zap.Error(err))
+		} else {
+			plog.Fatal(err)
+		}
+	}
+
 	_, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0)
 	if len(finishedCompactBytes) != 0 {
 		s.revMu.Lock()