Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat[work-in-progress]: add a PostgreSQL database as primary storage 🗃️ #40

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1 +1,13 @@
COMPOSE_PROJECT_NAME=cuttlink
# Docker Compose
COMPOSE_PROJECT_NAME=cuttlink

# Postgres Database
POSTGRES_USER=postgres
POSTGRES_PASSWORD=changeme
POSTGRES_DB=cuttlink
PGDATA=/var/lib/postgresql/data

# Postgres Database Admin
[email protected]
PGADMIN_DEFAULT_PASSWORD=admin
PGADMIN_LISTEN_PORT=80
11 changes: 11 additions & 0 deletions database/init-db.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
CREATE SCHEMA IF NOT EXISTS urls;
CREATE TABLE IF NOT EXISTS urls.guest (
ShortURL CHAR(7) PRIMARY KEY,
OriginalURL VARCHAR(500) NOT NULL,
CreationDate TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON SCHEMA urls IS 'Data related to urls.';
COMMENT ON TABLE urls.guest IS 'Table for urls related to guest users.';
COMMENT ON COLUMN urls.guest.ShortURL IS 'The short url a guest user gets.';
COMMENT ON COLUMN urls.guest.OriginalURL IS 'The long url a guest user wants to shortens.';
COMMENT ON COLUMN urls.guest.CreationDate IS 'The date and time with a timezone a guest user shortened the long url.';
13 changes: 13 additions & 0 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,23 @@ services:
build:
context: server
target: server-build
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_URL
restart: always

server-proxy:
restart: always

cacher:
restart: always

database:
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_DB
restart: unless-stopped

database-admin:
54 changes: 54 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ services:
build:
context: web
target: ui-dev
hostname: web
ports:
- '4200:4200'
networks:
- public
depends_on:
- server
labels:
cuttlink.description: 'Web Client'
volumes:
- ./web/:/app
- webnodemodules:/app/node_modules
Expand All @@ -20,48 +23,99 @@ services:
build:
context: server
target: server-dev
hostname: server
ports:
- '3000:3000'
networks:
- public
depends_on:
- database
- cacher
labels:
cuttlink.description: 'NodeJS Server'
volumes:
- ./server/:/app
- servernodemodules:/app/node_modules
environment:
WAIT_HOSTS: pgdb:5432, pgadmin:80
WAIT_LOGGER_LEVEL: debug
container_name: cuttlink_server

server-proxy:
build:
context: server
dockerfile: nginx/Dockerfile
hostname: server-proxy
ports:
- '8080:8080'
networks:
- public
depends_on:
- server
labels:
cuttlink.description: 'NGINX Proxy for Cuttlink server'
volumes:
- ./server/nginx/nginx.conf:/etc/nginx/nginx.conf
container_name: cuttlink_server_proxy

cacher:
build:
context: redis
hostname: cacher
ports:
- '6379:6379'
networks:
- public
labels:
cuttlink.description: 'Redis Cache'
volumes:
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf
- redisdata:/data
container_name: cache

database:
image: postgres:13-alpine
hostname: pgdb
ports:
- '5432:5432'
networks:
- public
- database
labels:
cuttlink.description: 'Postgres Database'
volumes:
- postgres-data:/var/lib/postgresql/data
- ./database/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
env_file:
- .env
container_name: pg_database

database-admin:
image: dpage/pgadmin4
hostname: pgadmin
ports:
- '5050:80'
networks:
- database
depends_on:
- database
labels:
cuttlink.description: 'Adminer Database'
volumes:
- postgresadmin:/var/lib/pgadmin
env_file:
- .env
container_name: pg_database_admin

networks:
public:
driver: bridge
database:
driver: bridge

volumes:
servernodemodules: {}
webnodemodules: {}
redisdata: {}
postgresdata: {}
postgresadmin: {}
3 changes: 3 additions & 0 deletions server/.eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
"es2021": true,
"node": true
},
"globals": {
"NodeJS": true
},
"extends": ["standard"],
"parser": "@typescript-eslint/parser",
"parserOptions": {
Expand Down
7 changes: 5 additions & 2 deletions server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
FROM node:14-slim AS base
WORKDIR /app
COPY ./package*.json ./
# wait for the database
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait
RUN chmod +x /wait

# dev environment
FROM base AS server-dev
Expand All @@ -11,7 +14,7 @@ COPY . .
ENV NODE_ENV development
EXPOSE 3000

ENTRYPOINT ["npm", "run", "dev"]
ENTRYPOINT ["sh", "-c", "/wait && npm run dev"]

# production environment
FROM base AS server-build
Expand All @@ -22,4 +25,4 @@ COPY . .
ENV NODE_ENV production
EXPOSE 3000

ENTRYPOINT ["npm", "start"]
ENTRYPOINT ["sh", "-c", "/wait && npm start"]
130 changes: 117 additions & 13 deletions server/src/config/cache.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,126 @@
import redis from 'redis'
import logger from './logger'
import { REDIS_BASE_URL } from '../config/common'
import { InitOptions } from '../types'
import { REDIS_BASE_URL as REDIS_URL } from './common'
import { redisError } from '../utils/ErrorHandlers'

const { REDIS_URL = REDIS_BASE_URL } = process.env
// Maximum delay between reconnection attempts after backoff
const maxReconnectDelay = 5000

const redisClient = redis.createClient({
url: REDIS_URL
})
const createRedisClient = (options: InitOptions = {}) => {
const { url, name } = options

const init = async () =>
new Promise((resolve, reject) => {
redisClient.on('connect', () => {
logger.info({ message: 'Redis client connected' })
resolve(redisClient)
logger.info({ message: 'Redis client resolved' })
// If redis url is not provided, bail out
if (!url) return

let pingInterval: NodeJS.Timeout
function stopPinging() {
pingInterval && clearInterval(pingInterval)
}

// Create the client
const client = redis.createClient({
url: url,
// Any running command that is unfulfilled when a connection is lost should
// NOT be retried after the connection has been reestablished.
retry_unfulfilled_commands: false,
// If we failed to send a new command during a disconnection, do NOT
// enqueue it to send later after the connection has been [re-]established.
enable_offline_queue: false,
// This timeout value will be applied to both the initial connection
// and any auto-reconnect attempts (if the `retry_strategy` option is
// provided). If not using the `retry_strategy` option, this value can be
// set to a very low number. If using the `retry_strategy` option to allow
// more than one reconnection attempt, this value must be set to a higher
// number. Defaults to 1 hour if not configured
connect_timeout: 60 * 60 * 1000, // 60 minutes
retry_strategy: function ({
attempt,
error,
total_retry_time: totalRetryTime,
times_connected: timesConnected
}) {
let delayPerAttempt = 100

// If the server appears to unavailable, slow down faster
if (
error &&
(error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND')
) {
delayPerAttempt *= 5
}

// Reconnect after delay
return Math.min(attempt * delayPerAttempt, maxReconnectDelay)
}
})

// If a `name` was provided, use it in the infix for logging event messages
const clientName = name ? `(${name})` : ''

client.on('connect', () => {
logger.info({ message: `Redis client ${clientName} connected` })
// Stop pinging the Redis server, if the timer already exists
stopPinging()

// Start pinging the server once per minute to prevent Redis connection
// from closing when its idle `timeout` configuration value expires
pingInterval = setInterval(() => {
client.ping(() => {})
}, 60 * 1000)
})

// Handle connection errors to prevent killing the Nodejs process
client.on('error', connectError => {
try {
// Forcibly close the connection to the Redis server
// Allow all still running commands to silently fail immediately
client.end(false)
} catch (disconnectError) {
// Swallow any failure
}

// Also, stop pinging the Redis server
stopPinging()
logger.error({ message: redisError(connectError, clientName) })
})

client.on('end', () => {
// Stop pinging the Redis server
stopPinging()
logger.debug({ message: `Redis client ${clientName} connection closed` })
})

client.on('ready', () => {
logger.info({
message: `Redis client ${clientName} ready to recieve commands`
})
})

redisClient.on('error', error => reject(error))
client.on('warning', msg => {
logger.warn({ message: `Redis client ${clientName} warning: ${msg}` })
})

export { init, redisClient }
client.on(
'reconnecting',
({
attempt,
delay,
error,
total_retry_time: totalRetryTime,
times_connected: timesConnected
}) => {
logger.alert({
message: `Redis client ${clientName} reconnecting, attempt ${attempt}, with ${delay} delay, due to ${redisError(
error
)}. Elapsed time: ${totalRetryTime}. Successful connections: ${timesConnected}.`
})
}
)

return client
}

const redisClient = createRedisClient({ url: REDIS_URL, name: 'node-js' })

export { createRedisClient, redisClient }
5 changes: 5 additions & 0 deletions server/src/config/custom-environment-variables.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,10 @@
"SENTRY_DSN": "SENTRY_DSN",
"REDIS_URL": {
"BASE_URL": "REDIS_URL"
},
"POSTGRES": {
"USER": "POSTGRES_USER",
"PASSWORD": "POSTGRES_PASSWORD",
"BASE_URL": "POSTGRES_URL"
}
}
6 changes: 6 additions & 0 deletions server/src/config/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,11 @@
"HOST": "cacher",
"PORT": "6379",
"BASE_URL": "redis://cacher:6379"
},
"POSTGRES": {
"USER": "postgres",
"PASSWORD": "changeme",
"// BASE_URL": "postgres://user:password@containerhostname:containerport/dbname",
"BASE_URL": "postgres://postgres:changeme@pgdb:5432/cuttlink"
}
}
5 changes: 0 additions & 5 deletions server/src/config/index.ts

This file was deleted.

6 changes: 4 additions & 2 deletions server/src/middlewares/security.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ import express, { Router } from 'express'
import rateLimit from 'express-rate-limit'
import RedisStore from 'rate-limit-redis'
import helmet from 'helmet'
import { redisClient } from '../config/cache'
import { HTTP429Error } from '../utils/httpErrors'
import { REDIS_BASE_URL as REDIS_URL } from '../config/common'
import { createRedisClient } from '../config/cache'

const isProduction = process.env.NODE_ENV === 'production'
const EXPIRES_IN_AS_SECONDS = 3 * 60
Expand All @@ -20,9 +21,10 @@ const handleRateLimit = (router: Router) => {
},
// the storage to use when persisting rate limit attempts
store: new RedisStore({
client: redisClient,
client: createRedisClient({ url: REDIS_URL, name: 'rate-limit' }),
// 3 mins in `s` (or practically unlimited outside production)
expiry: isProduction ? EXPIRES_IN_AS_SECONDS : 1,
// prefix to add to entries in Redis
prefix: 'rl:',
// @ts-ignore
// If Redis is not connected, let the request succeed as failover
Expand Down
Loading