cherry pick "etcd_cluster_version" metric" (#10257, #11233, #11254, #11265) to release-3.3

etcdserver/membership/cluster.go

@@ -36,6 +36,7 @@ import (
 	"github.com/coreos/etcd/version"
 
 	"github.com/coreos/go-semver/semver"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // RaftCluster is a list of Members that belong to the same raft cluster
@@ -368,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	} else {
 		plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String()))
 	}
+	oldVer := c.version
 	c.version = ver
 	mustDetectDowngrade(c.version)
 	if c.store != nil {
@@ -376,6 +378,10 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	if c.be != nil {
 		mustSaveClusterVersionToBackend(c.be, ver)
 	}
+	if oldVer != nil {
+		ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0)
+	}
+	ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1)
 	onSet(ver)
 }
 

etcdserver/membership/metrics.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	ClusterVersionMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "cluster",
+		Name:      "version",
+		Help:      "Which version is running. 1 for 'cluster_version' label with current cluster version",
+	},
+		[]string{"cluster_version"})
+)
+
+func init() {
+	prometheus.MustRegister(ClusterVersionMetrics)
+}

etcdserver/server.go

@@ -602,6 +602,7 @@ func (s *EtcdServer) start() {
 	s.leaderChanged = make(chan struct{})
 	if s.ClusterVersion() != nil {
 		plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String()))
+		membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1)
 	} else {
 		plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version)
 	}

tests/e2e/etcd_release_upgrade_test.go

@@ -103,6 +103,11 @@ func TestReleaseUpgrade(t *testing.T) {
 			}
 		}
 	}
+
+	// expect upgraded cluster version
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
 }
 
 func TestReleaseUpgradeWithRestart(t *testing.T) {

tests/e2e/metrics_test.go

@@ -0,0 +1,59 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/version"
+)
+
+func TestV3MetricsSecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "https"
+	testCtl(t, metricsTest)
+}
+
+func TestV3MetricsInsecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "http"
+	testCtl(t, metricsTest)
+}
+
+func metricsTest(cx ctlCtx) {
+	if err := ctlV3Put(cx, "k", "v", ""); err != nil {
+		cx.t.Fatal(err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: `etcd_debugging_mvcc_keys_total 1`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	ver := version.Version
+	if strings.HasSuffix(ver, "+git") {
+		ver = strings.Replace(ver, "+git", "", 1)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+}