diff --git a/src/checks/graphiteWorking.check.js b/src/checks/graphiteWorking.check.js index 6d8fd34..0e24a06 100644 --- a/src/checks/graphiteWorking.check.js +++ b/src/checks/graphiteWorking.check.js @@ -58,21 +58,31 @@ class GraphiteWorkingCheck extends Check { } const simplifiedResults = json.map(result => { - // This sums the number of nulls at the tail of the list of metrics. - const nullsForHowLong = result.datapoints.reduce((xs, x) => x[0] === null ? xs + 1 : 0, 0); - const simplifiedResult = { target: result.target, nullsForHowLong }; + + let nullsForHowManySeconds; + + if (result.datapoints.every(datapoint => datapoint[0] === null)) { + nullsForHowManySeconds = Infinity; + } else { + // This sums the number of seconds since the last non-null result at the tail of the list of metrics. + nullsForHowManySeconds = result.datapoints + .map((datapoint, index, array) => [datapoint[0], index > 0 ? datapoint[1] - array[index - 1][1] : 0]) + .reduce((xs, datapoint) => datapoint[0] === null ? xs + datapoint[1] : 0, 0); + } + + const simplifiedResult = { target: result.target, nullsForHowManySeconds }; log.info({ event: `${logEventPrefix}_NULLS_FOR_HOW_LONG` }, simplifiedResult); return simplifiedResult; }); - const failedResults = simplifiedResults.filter(r => r.nullsForHowLong > 2); + const failedResults = simplifiedResults.filter(r => r.nullsForHowManySeconds >= 180); if (failedResults.length === 0) { this.status = status.PASSED; this.checkOutput =`${this.metric} has data`; } else { this.status = status.FAILED; - this.checkOutput = failedResults.map(r => `${r.target} has been null for ${r.nullsForHowLong} minutes.`).join(' '); + this.checkOutput = failedResults.map(r => `${r.target} has been null for ${Math.round(r.nullsForHowManySeconds / 60)} minutes.`).join(' '); } }) .catch(err => { diff --git a/test/graphiteworking.spec.js b/test/graphiteworking.spec.js index 3e3bc13..2ff5e41 100644 --- a/test/graphiteworking.spec.js +++ b/test/graphiteworking.spec.js @@ -25,7 +25,7 @@ describe('Graphite Working Check', function(){ } ]; - const badResponse = [ + const recentlyBadResponse = [ { "target": "summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, \"1h\", \"sum\", true)", "datapoints": [ @@ -34,11 +34,25 @@ describe('Graphite Working Check', function(){ [ 1, 1459333260 ], [ null, 1459333320 ], [ null, 1459333380 ], - [ null, 1459333420 ], + [ null, 1459333440 ], ] } ]; + const completelyBadResponse = [ + { + "target": "summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, \"1h\", \"sum\", true)", + "datapoints": [ + [ null, 1459333140 ], + [ null, 1459333200 ], + [ null, 1459333260 ], + [ null, 1459333320 ], + [ null, 1459333380 ], + [ null, 1459333440 ], + ] + } + ]; + function waitFor(time){ return new Promise(resolve => setTimeout(resolve, time)); } @@ -75,8 +89,8 @@ describe('Graphite Working Check', function(){ }); }); - it('Should fail if there is no data', () => { - setup(badResponse); + it('Should fail if there is has been 2 or more minutes of missing data', () => { + setup(recentlyBadResponse); check.start(); return waitFor(10).then(() => { expect(check.getStatus().ok).to.be.false; @@ -84,6 +98,15 @@ describe('Graphite Working Check', function(){ }); }); + it('Should fail if there is no data', () => { + setup(completelyBadResponse); + check.start(); + return waitFor(10).then(() => { + expect(check.getStatus().ok).to.be.false; + expect(check.getStatus().checkOutput).to.equal('summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, "1h", "sum", true) has been null for Infinity minutes.'); + }); + }); + //todo get the graphite api key into the CI config - doesn't seem possible right now... describe.skip('Integration', function(){