From 511b673082bd72bbcb531aad4cfb300a6f1c81fb Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 22 Jan 2024 10:08:19 +0200 Subject: [PATCH 1/4] add health api endpoint to check health of API --- api.js | 2 + lib/api/health.js | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 lib/api/health.js diff --git a/api.js b/api.js index e5ca651b..66f2b213 100644 --- a/api.js +++ b/api.js @@ -47,6 +47,7 @@ const dkimRoutes = require('./lib/api/dkim'); const certsRoutes = require('./lib/api/certs'); const webhooksRoutes = require('./lib/api/webhooks'); const settingsRoutes = require('./lib/api/settings'); +const healthRoutes = require('./lib/api/health'); const { SettingsHandler } = require('./lib/settings-handler'); let userHandler; @@ -561,6 +562,7 @@ module.exports = done => { certsRoutes(db, server); webhooksRoutes(db, server); settingsRoutes(db, server, settingsHandler); + healthRoutes(db, server); if (process.env.NODE_ENV === 'test') { server.get( diff --git a/lib/api/health.js b/lib/api/health.js new file mode 100644 index 00000000..16235a1c --- /dev/null +++ b/lib/api/health.js @@ -0,0 +1,95 @@ +'use strict'; + +const Joi = require('joi'); +const tools = require('../tools'); +const { successRes } = require('../schemas/response/general-schemas'); + +module.exports = (db, server) => { + server.get( + { + path: '/health', + summary: 'Check the health of the API', + description: 'Check the status of the WildDuck API service, that is if db is connected and readable/writable, same for redis.', + tags: ['Health'], + validationObjs: { + requestBody: {}, + queryParams: {}, + pathParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ success: successRes }) + }, + 500: { + description: 'Failed', + model: Joi.object({ success: successRes, message: Joi.string().required().description('Error message specifying what went wrong') }) + } + } + } + }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const currentTimestamp = Date.now() / 1000; + + // 1) test that mongoDb is up + try { + const isConnected = await db.database.s.client.topology.isConnected(); + + if (!isConnected) { + res.status(500); + return res.json({ + success: false, + message: 'DB is down' + }); + } + } catch (err) { + res.status(500); + return res.json({ + success: false, + message: 'DB is down' + }); + } + + // 2) test that mongoDb is writeable + + try { + await db.database.collection(`${currentTimestamp}`).insert({ a: 'testWrite' }); + await db.database.collection(`${currentTimestamp}`).deleteOne({ a: 'testWrite' }); + } catch (err) { + res.status(500); + return res.json({ + success: false, + message: 'Could not write to DB' + }); + } + + // 3) test redis PING + db.redis.ping(err => { + if (err) { + res.status(500); + return res.json({ + success: false, + message: 'Redis is down' + }); + } + }); + + // 4) test if redis is writeable + try { + await db.redis.set(`${currentTimestamp}`, 'testVal'); + await db.redis.get(`${currentTimestamp}`); + await db.redis.del(`${currentTimestamp}`); + } catch (err) { + res.status(500); + return res.json({ + success: false, + message: 'Redis is not writeable/readable' + }); + } + + res.status(200); + return res.json({ success: true }); + }) + ); +}; From 7f4b05ebc5bd9a22c5581d6946d67ef8045644cc Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 22 Jan 2024 11:28:21 +0200 Subject: [PATCH 2/4] fixes and add graylog logging --- api.js | 2 +- lib/api/health.js | 53 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/api.js b/api.js index 66f2b213..0a9199f7 100644 --- a/api.js +++ b/api.js @@ -562,7 +562,7 @@ module.exports = done => { certsRoutes(db, server); webhooksRoutes(db, server); settingsRoutes(db, server, settingsHandler); - healthRoutes(db, server); + healthRoutes(db, server, loggelf); if (process.env.NODE_ENV === 'test') { server.get( diff --git a/lib/api/health.js b/lib/api/health.js index 16235a1c..44db6a88 100644 --- a/lib/api/health.js +++ b/lib/api/health.js @@ -4,7 +4,7 @@ const Joi = require('joi'); const tools = require('../tools'); const { successRes } = require('../schemas/response/general-schemas'); -module.exports = (db, server) => { +module.exports = (db, server, loggelf) => { server.get( { path: '/health', @@ -44,6 +44,10 @@ module.exports = (db, server) => { }); } } catch (err) { + loggelf({ + short_message: '[HEALTH] MongoDb is down. MongoDb is not connected.' + }); + res.status(500); return res.json({ success: false, @@ -54,9 +58,14 @@ module.exports = (db, server) => { // 2) test that mongoDb is writeable try { - await db.database.collection(`${currentTimestamp}`).insert({ a: 'testWrite' }); - await db.database.collection(`${currentTimestamp}`).deleteOne({ a: 'testWrite' }); + const insertData = await db.database.collection('health').insertOne({ [`${currentTimestamp}`]: 'testWrite' }); + await db.database.collection('health').deleteOne({ _id: insertData.insertedId }); } catch (err) { + loggelf({ + short_message: + '[HEALTH] could not write to MongoDb. MongoDB is not writeable, cannot write document to collection `health` and delete the document at that path.' + }); + res.status(500); return res.json({ success: false, @@ -65,22 +74,36 @@ module.exports = (db, server) => { } // 3) test redis PING - db.redis.ping(err => { - if (err) { - res.status(500); - return res.json({ - success: false, - message: 'Redis is down' - }); - } - }); + try { + await db.redis.ping(); + } catch (err) { + loggelf({ + short_message: '[HEALTH] Redis is down. PING to Redis failed.' + }); + + res.status(500); + return res.json({ + success: false, + message: 'Redis is down' + }); + } // 4) test if redis is writeable try { - await db.redis.set(`${currentTimestamp}`, 'testVal'); - await db.redis.get(`${currentTimestamp}`); - await db.redis.del(`${currentTimestamp}`); + await db.redis.hset('health', `${currentTimestamp}`, `${currentTimestamp}`); + const data = await db.redis.hget(`health`, `${currentTimestamp}`); + + if (data !== `${currentTimestamp}`) { + throw Error('Received data is not the same!'); + } + + await db.redis.hdel('health', `${currentTimestamp}`); } catch (err) { + loggelf({ + short_message: + '[HEALTH] Redis is not writeable/readable. Could not set hashkey `health` in redis, failed to get the key and/or delete the key.' + }); + res.status(500); return res.json({ success: false, From dac3c9b19d6504607625426fdbe7b2fdb2e17a6d Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 25 Jan 2024 10:21:01 +0200 Subject: [PATCH 3/4] round timestamp, cast to string. Use mongodb ping instead of topology.isConnected check --- lib/api/health.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/api/health.js b/lib/api/health.js index 44db6a88..8fb8603b 100644 --- a/lib/api/health.js +++ b/lib/api/health.js @@ -30,13 +30,13 @@ module.exports = (db, server, loggelf) => { tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const currentTimestamp = Date.now() / 1000; + const currentTimestamp = Math.round(Date.now() / 1000).toString(); // 1) test that mongoDb is up try { - const isConnected = await db.database.s.client.topology.isConnected(); + const pingResult = await db.database.command({ ping: 1 }); - if (!isConnected) { + if (!pingResult.ok) { res.status(500); return res.json({ success: false, @@ -45,7 +45,7 @@ module.exports = (db, server, loggelf) => { } } catch (err) { loggelf({ - short_message: '[HEALTH] MongoDb is down. MongoDb is not connected.' + short_message: '[HEALTH] MongoDb is down. MongoDb is not connected. PING not ok' }); res.status(500); From fa0e07713e0fb4810adede1de5d8fc920e011276 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 25 Jan 2024 12:40:08 +0200 Subject: [PATCH 4/4] add timeout to redis commands so that the health api endpoint will return a value --- lib/api/health.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/api/health.js b/lib/api/health.js index 8fb8603b..2b04b8e6 100644 --- a/lib/api/health.js +++ b/lib/api/health.js @@ -75,7 +75,8 @@ module.exports = (db, server, loggelf) => { // 3) test redis PING try { - await db.redis.ping(); + // Redis might try to reconnect causing a situation where given ping() command might never return a value, add a fixed timeout + await promiseRaceTimeoutWrapper(db.redis.ping(), 10000); } catch (err) { loggelf({ short_message: '[HEALTH] Redis is down. PING to Redis failed.' @@ -90,14 +91,15 @@ module.exports = (db, server, loggelf) => { // 4) test if redis is writeable try { - await db.redis.hset('health', `${currentTimestamp}`, `${currentTimestamp}`); - const data = await db.redis.hget(`health`, `${currentTimestamp}`); + await promiseRaceTimeoutWrapper(db.redis.hset('health', `${currentTimestamp}`, `${currentTimestamp}`), 10000); + + const data = await promiseRaceTimeoutWrapper(db.redis.hget(`health`, `${currentTimestamp}`), 10000); if (data !== `${currentTimestamp}`) { throw Error('Received data is not the same!'); } - await db.redis.hdel('health', `${currentTimestamp}`); + await promiseRaceTimeoutWrapper(db.redis.hdel('health', `${currentTimestamp}`), 10000); } catch (err) { loggelf({ short_message: @@ -116,3 +118,12 @@ module.exports = (db, server, loggelf) => { }) ); }; + +async function promiseRaceTimeoutWrapper(promise, timeout) { + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('Async call timed out!')), timeout); + }) + ]); +}