feat: prevent Nodejs server crashes when Redis server is unavailable ✨

Since we're going to setup a database as a primary data persistence, Redis won't be needed a primary memory store but just as a caching layer. So all redis connection errors are handled to prevent the errors from crashing the Nodejs server

* configure a retry strategy for redis client when connection to Redis server is unavailable
* ping Redis server every 60 minutes to prevent an idle connection timeout
* add event listeners to redis client for basic logging
* set log level to emergency on Nodejs server crashes
* add redis client names for Nodejs connection to Redis server and rate-limit store connection to Redis server
* format Redis connection errors to be more verbose and readable

Resolves #38

server/src/config/cache.ts

@@ -1,22 +1,126 @@
 import redis from 'redis'
 import logger from './logger'
-import { REDIS_BASE_URL } from '../config/common'
+import { InitOptions } from '../types'
+import { REDIS_BASE_URL as REDIS_URL } from './common'
+import { redisError } from '../utils/ErrorHandlers'
 
-const { REDIS_URL = REDIS_BASE_URL } = process.env
+// Maximum delay between reconnection attempts after backoff
+const maxReconnectDelay = 5000
 
-const redisClient = redis.createClient({
-  url: REDIS_URL
-})
+const createRedisClient = (options: InitOptions = {}) => {
+  const { url, name } = options
 
-const init = async () =>
-  new Promise((resolve, reject) => {
-    redisClient.on('connect', () => {
-      logger.info({ message: 'Redis client connected' })
-      resolve(redisClient)
-      logger.info({ message: 'Redis client resolved' })
+  // If redis url is not provided, bail out
+  if (!url) return
+
+  let pingInterval: NodeJS.Timeout
+  function stopPinging() {
+    pingInterval && clearInterval(pingInterval)
+  }
+
+  // Create the client
+  const client = redis.createClient({
+    url: url,
+    // Any running command that is unfulfilled when a connection is lost should
+    // NOT be retried after the connection has been reestablished.
+    retry_unfulfilled_commands: false,
+    // If we failed to send a new command during a disconnection, do NOT
+    // enqueue it to send later after the connection has been [re-]established.
+    enable_offline_queue: false,
+    // This timeout value will be applied to both the initial connection
+    // and any auto-reconnect attempts (if the `retry_strategy` option is
+    // provided). If not using the `retry_strategy` option, this value can be
+    // set to a very low number. If using the `retry_strategy` option to allow
+    // more than one reconnection attempt, this value must be set to a higher
+    // number. Defaults to 1 hour if not configured
+    connect_timeout: 60 * 60 * 1000, // 60 minutes
+    retry_strategy: function ({
+      attempt,
+      error,
+      total_retry_time: totalRetryTime,
+      times_connected: timesConnected
+    }) {
+      let delayPerAttempt = 100
+
+      // If the server appears to unavailable, slow down faster
+      if (
+        error &&
+        (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND')
+      ) {
+        delayPerAttempt *= 5
+      }
+
+      // Reconnect after delay
+      return Math.min(attempt * delayPerAttempt, maxReconnectDelay)
+    }
+  })
+
+  // If a `name` was provided, use it in the infix for logging event messages
+  const clientName = name ? `(${name})` : ''
+
+  client.on('connect', () => {
+    logger.info({ message: `Redis client ${clientName} connected` })
+    // Stop pinging the Redis server, if the timer already exists
+    stopPinging()
+
+    // Start pinging the server once per minute to prevent Redis connection
+    // from closing when its idle `timeout` configuration value expires
+    pingInterval = setInterval(() => {
+      client.ping(() => {})
+    }, 60 * 1000)
+  })
+
+  // Handle connection errors to prevent killing the Nodejs process
+  client.on('error', connectError => {
+    try {
+      // Forcibly close the connection to the Redis server
+      // Allow all still running commands to silently fail immediately
+      client.end(false)
+    } catch (disconnectError) {
+      // Swallow any failure
+    }
+
+    // Also, stop pinging the Redis server
+    stopPinging()
+    logger.error({ message: redisError(connectError, clientName) })
+  })
+
+  client.on('end', () => {
+    // Stop pinging the Redis server
+    stopPinging()
+    logger.debug({ message: `Redis client ${clientName} connection closed` })
+  })
+
+  client.on('ready', () => {
+    logger.info({
+      message: `Redis client ${clientName} ready to recieve commands`
     })
+  })
 
-    redisClient.on('error', error => reject(error))
+  client.on('warning', msg => {
+    logger.warn({ message: `Redis client ${clientName} warning: ${msg}` })
   })
 
-export { init, redisClient }
+  client.on(
+    'reconnecting',
+    ({
+      attempt,
+      delay,
+      error,
+      total_retry_time: totalRetryTime,
+      times_connected: timesConnected
+    }) => {
+      logger.alert({
+        message: `Redis client ${clientName} reconnecting, attempt ${attempt}, with ${delay} delay, due to ${redisError(
+          error
+        )}. Elapsed time: ${totalRetryTime}. Successful connections: ${timesConnected}.`
+      })
+    }
+  )
+
+  return client
+}
+
+const redisClient = createRedisClient({ url: REDIS_URL, name: 'node-js' })
+
+export { createRedisClient, redisClient }

server/src/middlewares/security.ts

@@ -2,8 +2,9 @@ import express, { Router } from 'express'
 import rateLimit from 'express-rate-limit'
 import RedisStore from 'rate-limit-redis'
 import helmet from 'helmet'
-import { redisClient } from '../config/cache'
 import { HTTP429Error } from '../utils/httpErrors'
+import { REDIS_BASE_URL as REDIS_URL } from '../config/common'
+import { createRedisClient } from '../config/cache'
 
 const isProduction = process.env.NODE_ENV === 'production'
 const EXPIRES_IN_AS_SECONDS = 3 * 60
@@ -20,9 +21,10 @@ const handleRateLimit = (router: Router) => {
     },
     // the storage to use when persisting rate limit attempts
     store: new RedisStore({
-      client: redisClient,
+      client: createRedisClient({ url: REDIS_URL, name: 'rate-limit' }),
       // 3 mins in `s` (or practically unlimited outside production)
       expiry: isProduction ? EXPIRES_IN_AS_SECONDS : 1,
+      // prefix to add to entries in Redis
       prefix: 'rl:',
       // @ts-ignore
       // If Redis is not connected, let the request succeed as failover

server/src/server.ts

@@ -4,19 +4,18 @@ import { applyMiddlewares, applyRoutes } from './utils'
 import middleware from './middlewares'
 import errorHandlers from './middlewares/errorHandlers'
 import routes from './services'
-import initDependencies from './config'
 import logger from './config/logger'
 
 process.on('uncaughtException', e => {
-  logger.error({
+  logger.emerg({
     message: 'uncaughtException',
     extra: e
   })
   process.exit(1)
 })
 
 process.on('unhandledRejection', e => {
-  logger.error({
+  logger.emerg({
     message: 'unhandledRejection',
     extra: e
   })
@@ -31,8 +30,7 @@ applyMiddlewares(errorHandlers, router)
 const { PORT = 3000 } = process.env
 const server = http.createServer(router)
 
-async function start() {
-  await initDependencies()
+function start() {
   server.listen(PORT, () =>
     logger.info({
       message: `Server running in ${process.env.NODE_ENV} mode on http://localhost:${PORT}...`

server/src/services/shorten/ShortenController.ts

@@ -5,6 +5,6 @@ export default async function getShortenedURL(url: string) {
   const random = new Random()
   const urlCode = random.randomsum()
 
-  redisClient.setex(urlCode, 3600, url)
+  redisClient!.setex(urlCode, 3600, url)
   return urlCode
 }

server/src/types/index.ts

@@ -14,4 +14,13 @@ interface Route {
   handler: Handler | Handler[]
 }
 
-export { Wrapper, Route }
+interface ErrorWithCode extends Error {
+  code?: string
+}
+
+type InitOptions = {
+  url?: string
+  name?: string
+}
+
+export { Wrapper, Route, ErrorWithCode, InitOptions }

server/src/utils/ErrorHandlers.ts

@@ -1,6 +1,7 @@
 import { Response, NextFunction } from 'express'
 import { HTTPClientError, HTTP404Error } from './httpErrors'
 import logger from '../config/logger'
+import { ErrorWithCode } from '../types'
 
 const notFoundError = () => {
   if (process.env.NODE_ENV === 'production') {
@@ -40,4 +41,17 @@ const serverError = (err: Error, res: Response, next: NextFunction) => {
   res.status(500).send(response)
 }
 
-export { notFoundError, clientError, serverError }
+const redisError = (error: ErrorWithCode, clientName?: string) => {
+  const errorCode = error ? error.code : null
+  const errorName = error ? error.constructor.name : 'Server disconnection'
+  const errorMsg = error
+    ? error.toString()
+    : 'unknown (commonly a server idle timeout)'
+  const preamble =
+    errorName +
+    `${clientName ?? ''}` +
+    (errorCode ? ` with code "${errorCode}"` : '')
+  return preamble + ': ' + errorMsg
+}
+
+export { notFoundError, clientError, serverError, redisError }