Merge pull request #95 from Financial-Times/lk/graphite-threshold-sum

Adding new check for graphite threshold sum

src/checks/graphiteSumThreshold.check.js

@@ -0,0 +1,76 @@
+'use strict';
+
+const logger = require('@financial-times/n-logger').default;
+const status = require('./status');
+const Check = require('./check');
+const fetch = require('node-fetch');
+const fetchres = require('fetchres');
+
+const logEventPrefix = 'GRAPHITE_SUM_THRESHOLD_CHECK';
+
+// Detects when the sum of all result values in a metric climbs above/below a threshold value
+
+class GraphiteSumThresholdCheck extends Check {
+
+	constructor (options) {
+		super(options);
+		this.threshold = options.threshold;
+		this.direction = options.direction || 'above';
+		this.from = options.from || '10min';
+		this.ftGraphiteBaseUrl = 'https://graphite-api.ft.com/render/?';
+        	this.ftGraphiteKey = process.env.FT_GRAPHITE_KEY;
+
+
+		if (!this.ftGraphiteKey) {
+			throw new Error('You must set FT_GRAPHITE_KEY environment variable');
+		}
+
+		if (!options.metric) {
+			throw new Error(`You must pass in a metric for the "${options.name}" check - e.g., "next.heroku.article.*.express.start"`);
+		}
+
+		if (!/next\./.test(options.metric)) {
+			throw new Error(`You must prepend the metric (${options.metric}) with "next." for the "${options.name}" check - e.g., "heroku.article.*.express.start" needs to be "next.heroku.article.*.express.start"`);
+        }
+
+		this.metric = options.metric;
+		this.sampleUrl = `${this.ftGraphiteBaseUrl}format=json&from=-${this.from}&target=${this.metric}`;
+		this.checkOutput = 'Graphite threshold check has not yet run';
+	}
+
+	tick () {
+
+		return fetch(this.sampleUrl, { headers: { key: this.ftGraphiteKey } })
+			.then(fetchres.json)
+			.then(results => {
+                const sum = this.sumResults(results);
+
+                if ((this.direction === 'above' && sum > this.threshold) ||
+                    (this.direction === 'below' && sum < this.threshold)) {
+                    this.status = status.FAILED;
+                    this.checkOutput = `Over the last ${this.from} the sum of results "${sum}" has moved ${this.direction} the threshold "${this.threshold}"`;
+                } else {
+                    this.status = status.PASSED;
+                    this.checkOutput = `Over the last ${this.from} the sum of results "${sum}" has not moved ${this.direction} the threshold "${this.threshold}"`;
+                }
+			})
+			.catch(err => {
+				logger.error({ event: `${logEventPrefix}_ERROR`, url: this.sampleUrl }, err);
+				this.status = status.FAILED;
+				this.checkOutput = 'Graphite threshold check failed to fetch data: ' + err.message;
+			});
+    }
+
+    sumResults (results) {
+        let sum = 0;
+        results.forEach((result) => {
+            result.datapoints.forEach((value) => {
+                sum += value[0] || 0;
+            });
+        });
+        return sum;
+    }
+
+}
+
+module.exports = GraphiteSumThresholdCheck;

src/checks/index.js

@@ -8,6 +8,7 @@ module.exports = {
 	pingdom : require('./pingdom.check'),
 	graphiteSpike: require('./graphiteSpike.check'),
 	graphiteThreshold: require('./graphiteThreshold.check'),
+	graphiteSumThreshold: require('./graphiteSumThreshold.check'),
 	graphiteWorking: require('./graphiteWorking.check'),
 	cloudWatchAlarm: require('./cloudWatchAlarm.check'),
 	cloudWatchThreshold: require('./cloudWatchThreshold.check'),

test/fixtures/config/graphiteSumThresholdFixture.js

@@ -0,0 +1,16 @@
+'use strict';
+module.exports = {
+	name: 'graphite',
+	descriptions : '',
+	checks : [
+		{
+			type: 'graphiteSumThreshold',
+			metric: 'next.metric.200',
+			name: 'test',
+			severity: 2,
+			businessImpact: 'catastrophic',
+			technicalSummary: 'god knows',
+			panicGuide: 'Don\'t Panic'
+		}
+	]
+};

test/graphiteSumThreshold.check.spec.js

@@ -0,0 +1,148 @@
+'use strict';
+
+const expect = require('chai').expect;
+const fixture = require('./fixtures/config/graphiteSumThresholdFixture').checks[0];
+const proxyquire = require('proxyquire').noCallThru().noPreserveCache();
+const sinon = require('sinon');
+
+describe('Graphite Sum Threshold Check', function(){
+
+	let check;
+
+	afterEach(function(){
+		check.stop();
+	});
+
+	context('Upper threshold enforced', function () {
+
+        it('Should be healthy if all datapoints summed are below upper threshold', function (done) {
+			const {Check} = mockGraphite([
+				{ datapoints: [[1, 1234567890], [2, 1234567891]] },
+				{ datapoints: [[3, 1234567892], [4, 1234567893]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 11
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.true;
+				done();
+			});
+		});
+
+		it('Should be healthy if all datapoints summed are equal to upper threshold', function (done) {
+			const {Check} = mockGraphite([
+				{ datapoints: [[1, 1234567890], [2, 1234567891]] },
+				{ datapoints: [[3, 1234567892], [4, 1234567893]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 10
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.true;
+				done();
+			});
+		});
+
+		it('should be unhealthy if all datapoints summed are above upper threshold', done => {
+			const {Check} = mockGraphite([
+				{ datapoints: [[1, 1234567890], [2, 1234567891]] },
+				{ datapoints: [[3, 1234567892], [4, 1234567893]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 9
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.false;
+				done();
+			});
+		});
+
+	});
+
+	context('Lower threshold enforced', function () {
+
+		it('Should be healthy if all datapoints summed are above lower threshold', function (done) {
+			const {Check} = mockGraphite([
+				{ datapoints: [[1, 1234567890], [2, 1234567891]] },
+				{ datapoints: [[3, 1234567892], [4, 1234567893]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 9,
+				direction: 'below'
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.true;
+				done();
+			});
+		});
+
+		it('Should be healthy if all datapoints summed are equal to lower threshold', function (done) {
+			const {Check} = mockGraphite([
+				{ datapoints: [[1, 1234567890], [2, 1234567891]] },
+				{ datapoints: [[3, 1234567892], [4, 1234567893]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 10,
+				direction: 'below'
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.true;
+				done();
+			});
+		});
+
+		it('should be unhealthy if all datapoints summed are below lower threshold', done => {
+			const {Check} = mockGraphite([
+				{ target: 'next.heroku.cpu.min', datapoints: [[1, 1234567890], [3, 1234567891]] },
+				{ target: 'next.heroku.disk.min', datapoints: [[2, 1234567890], [4, 1234567891]] }
+			]);
+			check = new Check(getCheckConfig({
+				threshold: 11,
+				direction: 'below'
+			}));
+			check.start();
+			setTimeout(() => {
+				expect(check.getStatus().ok).to.be.false;
+				done();
+			});
+		});
+
+	});
+
+	it('Should be possible to configure sample period', function(done){
+		const {Check, mockFetch} = mockGraphite([{ datapoints: [] }]);
+		check = new Check(getCheckConfig({
+			from: '24h'
+		}));
+		check.start();
+		setTimeout(() => {
+			expect(mockFetch.firstCall.args[0]).to.contain('from=-24h&target=next.metric.200');
+			done();
+		});
+	});
+
+});
+
+// Mocks a pair of calls to graphite for sample and baseline data
+function mockGraphite (results) {
+	const mockFetch = sinon.stub().returns(Promise.resolve({
+		status: 200,
+		ok: true,
+		json : () => Promise.resolve(results)
+	}));
+
+	return {
+        mockFetch,
+        Check: proxyquire('../src/checks/graphiteSumThreshold.check', {'node-fetch': mockFetch})
+    };
+}
+
+// Merge default fixture data with test config
+function getCheckConfig (conf) {
+	return Object.assign({}, fixture, conf || {});
+}