From f80c7b884cdab13be07dfcb7204e9c87682a91ae Mon Sep 17 00:00:00 2001 From: Liam Keaton Date: Wed, 11 Jul 2018 16:43:24 +0100 Subject: [PATCH 1/2] =?UTF-8?q?Adding=20new=20check=20for=20graphite=20thr?= =?UTF-8?q?eshold=20sum=20=20=F0=9F=90=BF=20v2.10.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/checks/graphiteSumThreshold.check.js | 76 +++++++++ src/checks/index.js | 1 + .../config/graphiteSumThresholdFixture.js | 16 ++ test/graphiteSumThreshold.check.spec.js | 148 ++++++++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 src/checks/graphiteSumThreshold.check.js create mode 100644 test/fixtures/config/graphiteSumThresholdFixture.js create mode 100644 test/graphiteSumThreshold.check.spec.js diff --git a/src/checks/graphiteSumThreshold.check.js b/src/checks/graphiteSumThreshold.check.js new file mode 100644 index 0000000..f0b91fc --- /dev/null +++ b/src/checks/graphiteSumThreshold.check.js @@ -0,0 +1,76 @@ +'use strict'; + +const logger = require('@financial-times/n-logger').default; +const status = require('./status'); +const Check = require('./check'); +const fetch = require('node-fetch'); +const fetchres = require('fetchres'); + +const logEventPrefix = 'GRAPHITE_SUM_THRESHOLD_CHECK'; + +// Detects when the sum of all result values in a metric climbs above/below a threshold value + +class GraphiteSumThresholdCheck extends Check { + + constructor(options){ + super(options); + this.threshold = options.threshold; + this.direction = options.direction || 'above'; + this.from = options.from || '10min'; + this.ftGraphiteBaseUrl = 'https://graphite-api.ft.com/render/?'; + this.ftGraphiteKey = process.env.FT_GRAPHITE_KEY; + + + if (!this.ftGraphiteKey) { + throw new Error('You must set FT_GRAPHITE_KEY environment variable'); + } + + if (!options.metric) { + throw new Error(`You must pass in a metric for the "${options.name}" check - e.g., "next.heroku.article.*.express.start"`); + } + + if (!/next\./.test(options.metric)) { + throw new Error(`You must prepend the metric (${options.metric}) with "next." for the "${options.name}" check - e.g., "heroku.article.*.express.start" needs to be "next.heroku.article.*.express.start"`); + } + + this.metric = options.metric; + this.sampleUrl = `${this.ftGraphiteBaseUrl}format=json&from=-${this.from}&target=${this.metric}`; + this.checkOutput = 'Graphite threshold check has not yet run'; + } + + tick(){ + + return fetch(this.sampleUrl, { headers: { key: this.ftGraphiteKey } }) + .then(fetchres.json) + .then(results => { + const sum = this.sumResults(results); + + if ((this.direction === 'above' && sum > this.threshold) || + (this.direction === 'below' && sum < this.threshold)) { + this.status = status.FAILED; + this.checkOutput = `Over the last ${this.from} the sum of results "${sum}" has moved ${this.direction} the threshold "${this.threshold}"`; + } else { + this.status = status.PASSED; + this.checkOutput = `Over the last ${this.from} the sum of results "${sum}" has not moved ${this.direction} the threshold "${this.threshold}"`; + } + }) + .catch(err => { + logger.error({ event: `${logEventPrefix}_ERROR`, url: this.sampleUrl }, err); + this.status = status.FAILED; + this.checkOutput = 'Graphite threshold check failed to fetch data: ' + err.message; + }); + } + + sumResults (results) { + let sum = 0; + results.forEach((result) => { + result.datapoints.forEach((value) => { + sum += value[0] || 0; + }); + }); + return sum; + } + +} + +module.exports = GraphiteSumThresholdCheck; diff --git a/src/checks/index.js b/src/checks/index.js index 891ec5d..cc0b06c 100644 --- a/src/checks/index.js +++ b/src/checks/index.js @@ -8,6 +8,7 @@ module.exports = { pingdom : require('./pingdom.check'), graphiteSpike: require('./graphiteSpike.check'), graphiteThreshold: require('./graphiteThreshold.check'), + graphiteSumThreshold: require('./graphiteSumThreshold.check'), graphiteWorking: require('./graphiteWorking.check'), cloudWatchAlarm: require('./cloudWatchAlarm.check'), cloudWatchThreshold: require('./cloudWatchThreshold.check'), diff --git a/test/fixtures/config/graphiteSumThresholdFixture.js b/test/fixtures/config/graphiteSumThresholdFixture.js new file mode 100644 index 0000000..d5c0d58 --- /dev/null +++ b/test/fixtures/config/graphiteSumThresholdFixture.js @@ -0,0 +1,16 @@ +'use strict'; +module.exports = { + name: 'graphite', + descriptions : '', + checks : [ + { + type: 'graphiteSumThreshold', + metric: 'next.metric.200', + name: 'test', + severity: 2, + businessImpact: 'catastrophic', + technicalSummary: 'god knows', + panicGuide: 'Don\'t Panic' + } + ] +}; diff --git a/test/graphiteSumThreshold.check.spec.js b/test/graphiteSumThreshold.check.spec.js new file mode 100644 index 0000000..755a835 --- /dev/null +++ b/test/graphiteSumThreshold.check.spec.js @@ -0,0 +1,148 @@ +'use strict'; + +const expect = require('chai').expect; +const fixture = require('./fixtures/config/graphiteSumThresholdFixture').checks[0]; +const proxyquire = require('proxyquire').noCallThru().noPreserveCache(); +const sinon = require('sinon'); + +describe('Graphite Sum Threshold Check', function(){ + + let check; + + afterEach(function(){ + check.stop(); + }); + + context('Upper threshold enforced', function () { + + it('Should be healthy if all datapoints summed are below upper threshold', function (done) { + const {Check} = mockGraphite([ + { datapoints: [[1, 1234567890], [2, 1234567891]] }, + { datapoints: [[3, 1234567892], [4, 1234567893]] } + ]); + check = new Check(getCheckConfig({ + threshold: 11 + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.true; + done(); + }); + }); + + it('Should be healthy if all datapoints summed are equal to upper threshold', function (done) { + const {Check} = mockGraphite([ + { datapoints: [[1, 1234567890], [2, 1234567891]] }, + { datapoints: [[3, 1234567892], [4, 1234567893]] } + ]); + check = new Check(getCheckConfig({ + threshold: 10 + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.true; + done(); + }); + }); + + it('should be unhealthy if all datapoints summed are above upper threshold', done => { + const {Check} = mockGraphite([ + { datapoints: [[1, 1234567890], [2, 1234567891]] }, + { datapoints: [[3, 1234567892], [4, 1234567893]] } + ]); + check = new Check(getCheckConfig({ + threshold: 9 + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.false; + done(); + }); + }); + + }); + + context('Lower threshold enforced', function () { + + it('Should be healthy if all datapoints summed are above lower threshold', function (done) { + const {Check} = mockGraphite([ + { datapoints: [[1, 1234567890], [2, 1234567891]] }, + { datapoints: [[3, 1234567892], [4, 1234567893]] } + ]); + check = new Check(getCheckConfig({ + threshold: 9, + direction: 'below' + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.true; + done(); + }); + }); + + it('Should be healthy if all datapoints summed are equal to lower threshold', function (done) { + const {Check} = mockGraphite([ + { datapoints: [[1, 1234567890], [2, 1234567891]] }, + { datapoints: [[3, 1234567892], [4, 1234567893]] } + ]); + check = new Check(getCheckConfig({ + threshold: 10, + direction: 'below' + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.true; + done(); + }); + }); + + it('should be unhealthy if all datapoints summed are below lower threshold', done => { + const {Check} = mockGraphite([ + { target: 'next.heroku.cpu.min', datapoints: [[1, 1234567890], [3, 1234567891]] }, + { target: 'next.heroku.disk.min', datapoints: [[2, 1234567890], [4, 1234567891]] } + ]); + check = new Check(getCheckConfig({ + threshold: 11, + direction: 'below' + })); + check.start(); + setTimeout(() => { + expect(check.getStatus().ok).to.be.false; + done(); + }); + }); + + }); + + it('Should be possible to configure sample period', function(done){ + const {Check, mockFetch} = mockGraphite([{ datapoints: [] }]); + check = new Check(getCheckConfig({ + from: '24h' + })); + check.start(); + setTimeout(() => { + expect(mockFetch.firstCall.args[0]).to.contain('from=-24h&target=next.metric.200'); + done(); + }); + }); + +}); + +// Mocks a pair of calls to graphite for sample and baseline data +function mockGraphite (results) { + const mockFetch = sinon.stub().returns(Promise.resolve({ + status: 200, + ok: true, + json : () => Promise.resolve(results) + })); + + return { + mockFetch, + Check: proxyquire('../src/checks/graphiteSumThreshold.check', {'node-fetch': mockFetch}) + }; +} + +// Merge default fixture data with test config +function getCheckConfig (conf) { + return Object.assign({}, fixture, conf || {}); +} \ No newline at end of file From 8071254d23a97db8dd62cb6b059b0cf2975d2f9c Mon Sep 17 00:00:00 2001 From: Liam Keaton Date: Wed, 11 Jul 2018 16:52:25 +0100 Subject: [PATCH 2/2] Fixing spaces --- src/checks/graphiteSumThreshold.check.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/checks/graphiteSumThreshold.check.js b/src/checks/graphiteSumThreshold.check.js index f0b91fc..010ce25 100644 --- a/src/checks/graphiteSumThreshold.check.js +++ b/src/checks/graphiteSumThreshold.check.js @@ -12,13 +12,13 @@ const logEventPrefix = 'GRAPHITE_SUM_THRESHOLD_CHECK'; class GraphiteSumThresholdCheck extends Check { - constructor(options){ + constructor (options) { super(options); this.threshold = options.threshold; this.direction = options.direction || 'above'; this.from = options.from || '10min'; this.ftGraphiteBaseUrl = 'https://graphite-api.ft.com/render/?'; - this.ftGraphiteKey = process.env.FT_GRAPHITE_KEY; + this.ftGraphiteKey = process.env.FT_GRAPHITE_KEY; if (!this.ftGraphiteKey) { @@ -38,7 +38,7 @@ class GraphiteSumThresholdCheck extends Check { this.checkOutput = 'Graphite threshold check has not yet run'; } - tick(){ + tick () { return fetch(this.sampleUrl, { headers: { key: this.ftGraphiteKey } }) .then(fetchres.json)