From 59a9a48fb524b932da946d9132f7b5850a75fd29 Mon Sep 17 00:00:00 2001 From: Dan Bamikiya Date: Wed, 18 Aug 2021 19:17:46 +0000 Subject: [PATCH] feat: prevent Nodejs server crashes when Redis server is unavailable :sparkles: Since we're going to setup a database as a primary data persistence, Redis won't be needed a primary memory store but just as a caching layer. So all redis connection errors are handled to prevent the errors from crashing the Nodejs server * configure a retry strategy for redis client when connection to Redis server is unavailable * ping Redis server every 60 minutes to prevent an idle connection timeout * add event listeners to redis client for basic logging * set log level to emergency on Nodejs server crashes * add redis client names for Nodejs connection to Redis server and rate-limit store connection to Redis server * format Redis connection errors to be more verbose and readable Resolves #38 --- server/src/config/cache.ts | 130 ++++++++++++++++-- server/src/config/index.ts | 5 - server/src/middlewares/security.ts | 6 +- server/src/server.ts | 8 +- .../src/services/shorten/ShortenController.ts | 2 +- server/src/types/index.ts | 11 +- server/src/utils/ErrorHandlers.ts | 16 ++- 7 files changed, 150 insertions(+), 28 deletions(-) delete mode 100644 server/src/config/index.ts diff --git a/server/src/config/cache.ts b/server/src/config/cache.ts index eca0f76..27a3cbf 100644 --- a/server/src/config/cache.ts +++ b/server/src/config/cache.ts @@ -1,22 +1,126 @@ import redis from 'redis' import logger from './logger' -import { REDIS_BASE_URL } from '../config/common' +import { InitOptions } from '../types' +import { REDIS_BASE_URL as REDIS_URL } from './common' +import { redisError } from '../utils/ErrorHandlers' -const { REDIS_URL = REDIS_BASE_URL } = process.env +// Maximum delay between reconnection attempts after backoff +const maxReconnectDelay = 5000 -const redisClient = redis.createClient({ - url: REDIS_URL -}) +const createRedisClient = (options: InitOptions = {}) => { + const { url, name } = options -const init = async () => - new Promise((resolve, reject) => { - redisClient.on('connect', () => { - logger.info({ message: 'Redis client connected' }) - resolve(redisClient) - logger.info({ message: 'Redis client resolved' }) + // If redis url is not provided, bail out + if (!url) return + + let pingInterval: NodeJS.Timeout + function stopPinging() { + pingInterval && clearInterval(pingInterval) + } + + // Create the client + const client = redis.createClient({ + url: url, + // Any running command that is unfulfilled when a connection is lost should + // NOT be retried after the connection has been reestablished. + retry_unfulfilled_commands: false, + // If we failed to send a new command during a disconnection, do NOT + // enqueue it to send later after the connection has been [re-]established. + enable_offline_queue: false, + // This timeout value will be applied to both the initial connection + // and any auto-reconnect attempts (if the `retry_strategy` option is + // provided). If not using the `retry_strategy` option, this value can be + // set to a very low number. If using the `retry_strategy` option to allow + // more than one reconnection attempt, this value must be set to a higher + // number. Defaults to 1 hour if not configured + connect_timeout: 60 * 60 * 1000, // 60 minutes + retry_strategy: function ({ + attempt, + error, + total_retry_time: totalRetryTime, + times_connected: timesConnected + }) { + let delayPerAttempt = 100 + + // If the server appears to unavailable, slow down faster + if ( + error && + (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND') + ) { + delayPerAttempt *= 5 + } + + // Reconnect after delay + return Math.min(attempt * delayPerAttempt, maxReconnectDelay) + } + }) + + // If a `name` was provided, use it in the infix for logging event messages + const clientName = name ? `(${name})` : '' + + client.on('connect', () => { + logger.info({ message: `Redis client ${clientName} connected` }) + // Stop pinging the Redis server, if the timer already exists + stopPinging() + + // Start pinging the server once per minute to prevent Redis connection + // from closing when its idle `timeout` configuration value expires + pingInterval = setInterval(() => { + client.ping(() => {}) + }, 60 * 1000) + }) + + // Handle connection errors to prevent killing the Nodejs process + client.on('error', connectError => { + try { + // Forcibly close the connection to the Redis server + // Allow all still running commands to silently fail immediately + client.end(false) + } catch (disconnectError) { + // Swallow any failure + } + + // Also, stop pinging the Redis server + stopPinging() + logger.error({ message: redisError(connectError, clientName) }) + }) + + client.on('end', () => { + // Stop pinging the Redis server + stopPinging() + logger.debug({ message: `Redis client ${clientName} connection closed` }) + }) + + client.on('ready', () => { + logger.info({ + message: `Redis client ${clientName} ready to recieve commands` }) + }) - redisClient.on('error', error => reject(error)) + client.on('warning', msg => { + logger.warn({ message: `Redis client ${clientName} warning: ${msg}` }) }) -export { init, redisClient } + client.on( + 'reconnecting', + ({ + attempt, + delay, + error, + total_retry_time: totalRetryTime, + times_connected: timesConnected + }) => { + logger.alert({ + message: `Redis client ${clientName} reconnecting, attempt ${attempt}, with ${delay} delay, due to ${redisError( + error + )}. Elapsed time: ${totalRetryTime}. Successful connections: ${timesConnected}.` + }) + } + ) + + return client +} + +const redisClient = createRedisClient({ url: REDIS_URL, name: 'node-js' }) + +export { createRedisClient, redisClient } diff --git a/server/src/config/index.ts b/server/src/config/index.ts deleted file mode 100644 index dec795c..0000000 --- a/server/src/config/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { init as initCache } from './cache' - -export default async function initDependencies() { - await initCache() -} diff --git a/server/src/middlewares/security.ts b/server/src/middlewares/security.ts index 33cac1c..b45629c 100644 --- a/server/src/middlewares/security.ts +++ b/server/src/middlewares/security.ts @@ -2,8 +2,9 @@ import express, { Router } from 'express' import rateLimit from 'express-rate-limit' import RedisStore from 'rate-limit-redis' import helmet from 'helmet' -import { redisClient } from '../config/cache' import { HTTP429Error } from '../utils/httpErrors' +import { REDIS_BASE_URL as REDIS_URL } from '../config/common' +import { createRedisClient } from '../config/cache' const isProduction = process.env.NODE_ENV === 'production' const EXPIRES_IN_AS_SECONDS = 3 * 60 @@ -20,9 +21,10 @@ const handleRateLimit = (router: Router) => { }, // the storage to use when persisting rate limit attempts store: new RedisStore({ - client: redisClient, + client: createRedisClient({ url: REDIS_URL, name: 'rate-limit' }), // 3 mins in `s` (or practically unlimited outside production) expiry: isProduction ? EXPIRES_IN_AS_SECONDS : 1, + // prefix to add to entries in Redis prefix: 'rl:', // @ts-ignore // If Redis is not connected, let the request succeed as failover diff --git a/server/src/server.ts b/server/src/server.ts index 4135feb..598bedf 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -4,11 +4,10 @@ import { applyMiddlewares, applyRoutes } from './utils' import middleware from './middlewares' import errorHandlers from './middlewares/errorHandlers' import routes from './services' -import initDependencies from './config' import logger from './config/logger' process.on('uncaughtException', e => { - logger.error({ + logger.emerg({ message: 'uncaughtException', extra: e }) @@ -16,7 +15,7 @@ process.on('uncaughtException', e => { }) process.on('unhandledRejection', e => { - logger.error({ + logger.emerg({ message: 'unhandledRejection', extra: e }) @@ -31,8 +30,7 @@ applyMiddlewares(errorHandlers, router) const { PORT = 3000 } = process.env const server = http.createServer(router) -async function start() { - await initDependencies() +function start() { server.listen(PORT, () => logger.info({ message: `Server running in ${process.env.NODE_ENV} mode on http://localhost:${PORT}...` diff --git a/server/src/services/shorten/ShortenController.ts b/server/src/services/shorten/ShortenController.ts index 21e7123..789fd4b 100644 --- a/server/src/services/shorten/ShortenController.ts +++ b/server/src/services/shorten/ShortenController.ts @@ -5,6 +5,6 @@ export default async function getShortenedURL(url: string) { const random = new Random() const urlCode = random.randomsum() - redisClient.setex(urlCode, 3600, url) + redisClient!.setex(urlCode, 3600, url) return urlCode } diff --git a/server/src/types/index.ts b/server/src/types/index.ts index e1d9bbe..518dd56 100644 --- a/server/src/types/index.ts +++ b/server/src/types/index.ts @@ -14,4 +14,13 @@ interface Route { handler: Handler | Handler[] } -export { Wrapper, Route } +interface ErrorWithCode extends Error { + code?: string +} + +type InitOptions = { + url?: string + name?: string +} + +export { Wrapper, Route, ErrorWithCode, InitOptions } diff --git a/server/src/utils/ErrorHandlers.ts b/server/src/utils/ErrorHandlers.ts index f6f1794..f32731d 100644 --- a/server/src/utils/ErrorHandlers.ts +++ b/server/src/utils/ErrorHandlers.ts @@ -1,6 +1,7 @@ import { Response, NextFunction } from 'express' import { HTTPClientError, HTTP404Error } from './httpErrors' import logger from '../config/logger' +import { ErrorWithCode } from '../types' const notFoundError = () => { if (process.env.NODE_ENV === 'production') { @@ -40,4 +41,17 @@ const serverError = (err: Error, res: Response, next: NextFunction) => { res.status(500).send(response) } -export { notFoundError, clientError, serverError } +const redisError = (error: ErrorWithCode, clientName?: string) => { + const errorCode = error ? error.code : null + const errorName = error ? error.constructor.name : 'Server disconnection' + const errorMsg = error + ? error.toString() + : 'unknown (commonly a server idle timeout)' + const preamble = + errorName + + `${clientName ?? ''}` + + (errorCode ? ` with code "${errorCode}"` : '') + return preamble + ': ' + errorMsg +} + +export { notFoundError, clientError, serverError, redisError }