Merge pull request #105 from Financial-Times/ops-cops-correct-graphit…

…e-working-check Make graphiteWorkingCheck look at timestamps
Financial-Times · Nov 28, 2018 · 4a3ec77 · 4a3ec77
2 parents c475af5 + a3a55b5
commit 4a3ec77
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 9 deletions.
diff --git a/src/checks/graphiteWorking.check.js b/src/checks/graphiteWorking.check.js
@@ -58,21 +58,31 @@ class GraphiteWorkingCheck extends Check {
 				}
 
 				const simplifiedResults = json.map(result => {
-					// This sums the number of nulls at the tail of the list of metrics.
-					const nullsForHowLong = result.datapoints.reduce((xs, x) => x[0] === null ? xs + 1 : 0, 0);
-					const simplifiedResult = { target: result.target, nullsForHowLong };
+
+					let nullsForHowManySeconds;
+
+					if (result.datapoints.every(datapoint => datapoint[0] === null)) {
+						nullsForHowManySeconds = Infinity;
+					} else {
+						// This sums the number of seconds since the last non-null result at the tail of the list of metrics.
+                        nullsForHowManySeconds = result.datapoints
+                            .map((datapoint, index, array) => [datapoint[0], index > 0 ? datapoint[1] - array[index - 1][1]  : 0])
+                            .reduce((xs, datapoint) => datapoint[0] === null ? xs + datapoint[1] : 0, 0);
+                    }
+
+					const simplifiedResult = { target: result.target, nullsForHowManySeconds };
 					log.info({ event: `${logEventPrefix}_NULLS_FOR_HOW_LONG` }, simplifiedResult);
 					return simplifiedResult;
 				});
 
-				const failedResults = simplifiedResults.filter(r => r.nullsForHowLong > 2);
+				const failedResults = simplifiedResults.filter(r => r.nullsForHowManySeconds >= 180);
 
 				if (failedResults.length === 0) {
 					this.status = status.PASSED;
 					this.checkOutput =`${this.metric} has data`;
 				} else {
 					this.status = status.FAILED;
-					this.checkOutput = failedResults.map(r => `${r.target} has been null for ${r.nullsForHowLong} minutes.`).join(' ');
+					this.checkOutput = failedResults.map(r => `${r.target} has been null for ${Math.round(r.nullsForHowManySeconds / 60)} minutes.`).join(' ');
 				}
 			})
 			.catch(err => {

diff --git a/test/graphiteworking.spec.js b/test/graphiteworking.spec.js
@@ -25,7 +25,7 @@ describe('Graphite Working Check', function(){
 		}
 	];
 
-	const badResponse = [
+	const recentlyBadResponse = [
 		{
 			"target": "summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, \"1h\", \"sum\", true)",
 			"datapoints": [
@@ -34,11 +34,25 @@ describe('Graphite Working Check', function(){
 				[ 1, 1459333260 ],
 				[ null, 1459333320 ],
 				[ null, 1459333380 ],
-				[ null, 1459333420 ],
+				[ null, 1459333440 ],
 			]
 		}
 	];
 
+    const completelyBadResponse = [
+        {
+            "target": "summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, \"1h\", \"sum\", true)",
+            "datapoints": [
+                [ null, 1459333140 ],
+                [ null, 1459333200 ],
+                [ null, 1459333260 ],
+                [ null, 1459333320 ],
+                [ null, 1459333380 ],
+                [ null, 1459333440 ],
+            ]
+        }
+    ];
+
 	function waitFor(time){
 		return new Promise(resolve => setTimeout(resolve, time));
 	}
@@ -75,15 +89,24 @@ describe('Graphite Working Check', function(){
 		});
 	});
 
-	it('Should fail if there is no data', () => {
-		setup(badResponse);
+	it('Should fail if there is has been 2 or more minutes of missing data', () => {
+		setup(recentlyBadResponse);
 		check.start();
 		return waitFor(10).then(() => {
 			expect(check.getStatus().ok).to.be.false;
 			expect(check.getStatus().checkOutput).to.equal('summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, "1h", "sum", true) has been null for 3 minutes.');
 		});
 	});
 
+    it('Should fail if there is no data', () => {
+        setup(completelyBadResponse);
+        check.start();
+        return waitFor(10).then(() => {
+            expect(check.getStatus().ok).to.be.false;
+            expect(check.getStatus().checkOutput).to.equal('summarize(next.fastly.133g5BGAc00Hv4v8t0dMry.anzac.requests, "1h", "sum", true) has been null for Infinity minutes.');
+        });
+    });
+
 	//todo get the graphite api key into the CI config - doesn't seem possible right now...
 	describe.skip('Integration', function(){