diff --git a/Makefile b/Makefile index ed68c1ae..e6919c0f 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,8 @@ REMOVE_ANSI_FLAG := $(if $(filter 1,$(DISABLE_ANSI)),,--ansi never) DOCKER_COMPOSE_COMMAND=docker compose $(REMOVE_ANSI_FLAG) -p bhasai +setup-barman: + @./scripts/setup-barman.sh install-docker: @./scripts/install-docker.sh diff --git a/common/db/Dockerfile b/common/db/Dockerfile new file mode 100644 index 00000000..4e4d5709 --- /dev/null +++ b/common/db/Dockerfile @@ -0,0 +1,55 @@ +FROM samagragovernance/postgres:1.0.1-pg15 + +ARG BARMAN_SSH_PUBLIC_KEY +ARG POSTGRES_SSH_PUBLIC_KEY +ARG POSTGRES_SSH_PRIVATE_KEY +ARG BARMAN_HOST + +ENV BARMAN_HOST=$BARMAN_HOST + +ADD config/postgresql.conf.template /etc/postgresql/postgresql.conf.template +ADD config/pg_hba.conf.template /etc/postgresql/pg_hba.conf.template + +RUN apk update && \ + apk add envsubst rsync && \ + envsubst < /etc/postgresql/postgresql.conf.template > /etc/postgresql/postgresql.conf && \ + envsubst < /etc/postgresql/pg_hba.conf.template > /etc/postgresql/pg_hba.conf; + + +# Install OpenSSH +RUN apk add --update --no-cache openssh openssh-keygen + +RUN ssh-keygen -A + +# Verify that the host keys exist +RUN ls -l /etc/ssh/ssh_host_* + +# Enable SSH and configure key-based authentication +RUN mkdir -p /root/.ssh && chmod 700 /root/.ssh + +# Use build argument to add the public key +RUN echo "$BARMAN_SSH_PUBLIC_KEY" > /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys + +# Create the .ssh directory for the postgres user +RUN mkdir -p /var/lib/postgresql/.ssh && \ + chmod 700 /var/lib/postgresql/.ssh && \ + chown postgres:postgres /var/lib/postgresql/.ssh + +# Use build argument to add the public key for the postgres user +RUN echo "$BARMAN_SSH_PUBLIC_KEY" > /var/lib/postgresql/.ssh/authorized_keys && \ + chmod 600 /var/lib/postgresql/.ssh/authorized_keys && \ + chown postgres:postgres /var/lib/postgresql/.ssh/authorized_keys + + +# Configure SSH daemon +RUN echo 'PasswordAuthentication no' >> /etc/ssh/sshd_config +RUN echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config +RUN echo 'AllowUsers postgres' >> /etc/ssh/sshd_config + +RUN chown -R postgres:postgres /etc/ssh + +USER postgres + +RUN echo "$POSTGRES_SSH_PUBLIC_KEY" > /var/lib/postgresql/.ssh/id_ed25519.pub && chmod 600 /var/lib/postgresql/.ssh/id_ed25519.pub +RUN echo "$POSTGRES_SSH_PRIVATE_KEY" > /var/lib/postgresql/.ssh/id_ed25519 && chmod 600 /var/lib/postgresql/.ssh/id_ed25519 +RUN ssh-keyscan -H "$BARMAN_HOST" >> /var/lib/postgresql/.ssh/known_hosts diff --git a/common/db/README.md b/common/db/README.md new file mode 100644 index 00000000..f2863f80 --- /dev/null +++ b/common/db/README.md @@ -0,0 +1,13 @@ +# Steps to run after the db container is started + +1. Run `docker exec db /usr/sbin/sshd` + +# Steps to setup barman + +1. Run `make setup-barman` + +# Useful Commands + +`barman check mydb` to check the status of mydb +`barman backup mydb` to backup mydb +`barman switch-xlog --force --archive mydb` to fix wal issues \ No newline at end of file diff --git a/common/db/config/pg_hba.conf.template b/common/db/config/pg_hba.conf.template new file mode 100644 index 00000000..876e2c45 --- /dev/null +++ b/common/db/config/pg_hba.conf.template @@ -0,0 +1,6 @@ +# TYPE DATABASE USER ADDRESS METHOD +local all all trust +host all all localhost trust +host replication streaming_barman 0.0.0.0/0 md5 +host all barman 0.0.0.0/0 md5 + diff --git a/common/db/config/postgresql.conf.template b/common/db/config/postgresql.conf.template new file mode 100644 index 00000000..0effa208 --- /dev/null +++ b/common/db/config/postgresql.conf.template @@ -0,0 +1,4 @@ +listen_addresses = '*' +wal_level = replica +archive_mode = on +archive_command = 'rsync -a %p barman@${BARMAN_HOST}:/var/lib/barman/mydb/streaming/%f' diff --git a/common/db/docker-compose.yaml b/common/db/docker-compose.yaml new file mode 100644 index 00000000..6d7a7aac --- /dev/null +++ b/common/db/docker-compose.yaml @@ -0,0 +1,29 @@ +services: + db: + build: + context: ./ + dockerfile: Dockerfile + args: + ENABLE_BARMAN: ${ENABLE_BARMAN:-false} + BARMAN_HOST: ${BARMAN_HOST} + DB_SSH_PUBLIC_KEY: ${DB_SSH_PUBLIC_KEY} + DB_SSH_PRIVATE_KEY: ${DB_SSH_PRIVATE_KEY} + restart: always + volumes: + - db:/var/lib/postgresql/data + ports: + - "5432:5432/tcp" + - "2222:22/tcp" + command: -c 'config_file=/etc/postgresql/postgresql.conf' -c 'hba_file=/etc/postgresql/pg_hba.conf' + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] + interval: 5s + timeout: 5s + retries: 5 + environment: + POSTGRES_USER: ${POSTGRES_USER:?POSTGRES_USER is not set in .env} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set in .env} + +volumes: + db: + diff --git a/common/db/entrypoint.sh b/common/db/entrypoint.sh new file mode 100755 index 00000000..e719eb5b --- /dev/null +++ b/common/db/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Start PostgreSQL +pg_ctl -D "$PGDATA" -o "-c archive_mode=on -c archive_command='test ! -f /var/lib/postgresql/archive/%f && cp %p /var/lib/postgresql/archive/%f'" start + +# Start SSH +exec /usr/sbin/sshd -D -e diff --git a/common/sample.env b/common/sample.env index 2789a8fe..bd43987c 100644 --- a/common/sample.env +++ b/common/sample.env @@ -4,6 +4,17 @@ MINIO_ROOT_PASSWORD= MINIO_ACCESS_KEY= MINIO_SECRET_KEY= + +# For db service +# NOTE: Refer to common/db/README.md to generate DB_SSH_PRIVATE_KEY and DB_SSH_PUBLIC_KEY if you set ENABLE_BARMAN=tru +# default value is false, set it to true to enable barman +ENABLE_BARMAN= +BARMAN_HOST= +DB_SSH_PRIVATE_KEY= +DB_SSH_PUBLIC_KEY= +POSTGRES_USER= +POSTGRES_PASSWORD= + # Set your organization name org= @@ -35,4 +46,4 @@ FUSIONAUTH_POSTGRES_PASSWORD= FUSIONAUTH_APP_RUNTIME_MODE=development FUSIONAUTH_API_KEY= FUSIONAUTH_ADMIN_EMAIL= -FUSIONAUTH_ADMIN_PASSWORD= \ No newline at end of file +FUSIONAUTH_ADMIN_PASSWORD= diff --git a/docker-compose.yaml.example b/docker-compose.yaml.example index 01627f18..85aa6862 100644 --- a/docker-compose.yaml.example +++ b/docker-compose.yaml.example @@ -3,6 +3,7 @@ include: # - ./common/minio/docker-compose.yaml # - ./common/environment/docker-compose.yaml # - ./common/fusionauth/docker-compose.yaml +# - ./common/db/docker-compose.yaml - ./common/registry/docker-compose.yaml diff --git a/scripts/setup-barman.sh b/scripts/setup-barman.sh new file mode 100755 index 00000000..34e78550 --- /dev/null +++ b/scripts/setup-barman.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +echo "Enter hostname/fqdn of postgres server:" +read host_name +echo "Enter database name to replicate wals:" +read db_name +echo "Enter password for barman user" +read barman_password +echo "Enter password for streaming_barman user" +read streaming_barman_password + +echo "Entered hostname is $host_name and database name is $db_name" + +### Function to confirm continuation +prompt_continue() { + while true; do + read -p "Do you want to continue? (yes/no): " yn + case $yn in + [Yy]* ) + echo "Continuing the script..." + break + ;; + [Nn]* ) + echo "Exiting the script..." + exit 0 + ;; + * ) + echo "Please answer yes or no." + ;; + esac + done +} +prompt_continue + +### Update and install required packages if not already installed +echo "Updating package list..." +apt-get update +if ! dpkg -l | grep -qw curl; then + echo "Installing curl..." + apt-get install -y curl +else + echo "curl is already installed, skipping........." +fi +if ! dpkg -l | grep -qw ca-certificates; then + echo "Installing ca-certificates..." + apt-get install -y ca-certificates +else + echo "ca-certificates is already installed ,skipping.........." +fi +if ! dpkg -l | grep -qw gnupg; then + echo "Installing gnupg..." + apt-get install -y gnupg +else + echo "gnupg is already installed, skipping ............." +fi + +### Add PostgreSQL's authentication key if not already added +if ! apt-key list | grep -qw ACCC4CF8; then + echo "Adding PostgreSQL's authentication key..." + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +else + echo "PostgreSQL's authentication key already added, skippping..........." +fi + +### Add PostgreSQL repository if not already added +if [ ! -f /etc/apt/sources.list.d/pgdg.list ]; then + echo "Adding PostgreSQL repository..." + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + apt-get update +else + echo "PostgreSQL repository already added, skipping........." +fi + +### Install barman if not already installed +if ! dpkg -l | grep -qw barman; then + echo "Installing barman..." + apt-get -y install barman +else + echo "barman is already installed, skipping.........." +fi + +# Create barman configuration file +config_file="/etc/barman.d/$host_name.conf" +if [ -e $config_file ]; then + echo "Configuration file $config_file exists, deleting and recreating..." + rm -f $config_file +else + echo "Generating barman configuration file $config_file for streaming backup of database..." +fi + +cat < /etc/barman.conf +[barman] +barman_home = /backup/barman +barman_user = barman +log_file = /var/log/barman/barman.log +compression = gzip +reuse_backup = link +backup_method = rsync +archiver = on +EOF + +cat < $config_file +[$host_name] +description = "Main PostgreSQL Database" +conninfo = host=$host_name user=barman dbname=$db_name password=$barman_password +ssh_command = ssh -q postgres@$host_name -p 2222 +retention_policy_mode = auto +retention_policy = RECOVERY WINDOW OF 7 days +wal_retention_policy = main +EOF + +echo "Configuration file $config_file created." + +### Create .pgpass file for barman user +barman_home=$(getent passwd barman | cut -d':' -f6) +pgpass_file="$barman_home/.pgpass" +if [ -e $pgpass_file ]; then + echo "$pgpass_file exists, deleting and recreating..." + rm -f $pgpass_file +else + echo "Creating $pgpass_file for credentials..." +fi + +sudo -u barman bash -c "echo '$host_name:5432:replication:barman:$barman_password' > ~/.pgpass" +sudo -u barman bash -c "echo '$host_name:5432:replication:streaming_barman:$streaming_barman_password' >> ~/.pgpass" +sudo -u barman bash -c "chmod 600 ~/.pgpass" +echo ".pgpass file created and permissions set." + + +echo "Barman Installation Completed"