backend: add prometheus metric for large snapshot duration.

FIXES #7878
etcd-io · May 6, 2017 · 230106d · 230106d
1 parent db6f45e
commit 230106d
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/mvcc/backend/backend.go b/mvcc/backend/backend.go
@@ -176,6 +176,7 @@ func (b *backend) Snapshot() Snapshot {
 			case <-ticker.C:
 				plog.Warningf("snapshotting is taking more than %v seconds to finish [started at %v]", time.Since(start).Seconds(), start)
 			case <-stopc:
+				snapshotDurations.Observe(time.Since(start).Seconds())
 				return
 			}
 		}

diff --git a/mvcc/backend/metrics.go b/mvcc/backend/metrics.go
@@ -24,8 +24,18 @@ var (
 		Help:      "The latency distributions of commit called by backend.",
 		Buckets:   prometheus.ExponentialBuckets(0.001, 2, 14),
 	})
+
+	snapshotDurations = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "etcd",
+		Subsystem: "disk",
+		Name:      "backend_snapshot_duration_seconds",
+		Help:      "The latency distribution of backend snapshots.",
+		// 10 ms -> 655 seconds
+		Buckets: prometheus.ExponentialBuckets(.01, 2, 17),
+	})
 )
 
 func init() {
 	prometheus.MustRegister(commitDurations)
+	prometheus.MustRegister(snapshotDurations)
 }