Merge pull request #1 from silinternational/develop

Inital commit

.gitignore

@@ -0,0 +1,11 @@
+# Vagrant folder
+.vagrant/
+
+# phpstorm project files
+.idea/
+/nbproject/
+# local config files
+local.env
+
+vendor/
+*.aes

Dockerfile

@@ -0,0 +1,22 @@
+FROM alpine:3.6
+
+RUN apk update \
+ && apk add --no-cache rsyslog rsyslog-tls \
+                       ca-certificates openssl \
+                       bash \
+                       postgresql \
+                       postgresql-client \
+                       python py-pip \
+ && update-ca-certificates \
+ && pip install s3cmd python-magic
+
+COPY dockerbuild/rsyslog.conf /etc/rsyslog.conf
+
+RUN wget https://raw.githubusercontent.com/silinternational/runny/0.2/runny -O /usr/local/bin/runny \
+ && chmod +x /usr/local/bin/runny
+
+COPY application/ /data/
+WORKDIR /data
+
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["crond -f"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 SIL International
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,14 @@
+start: restore backup
+
+restore: db
+	docker-compose up -d restore
+
+backup: db
+	docker-compose up -d backup
+
+db:
+	docker-compose up -d db adminer
+
+clean:
+	docker-compose kill
+	docker system prune -f

application/backup.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+
+for dbName in ${DB_NAMES}; do
+    logger -p user.info "backing up ${dbName}..."
+
+    start=$(date +%s)
+    runny $(PGPASSWORD=${DB_PASSWORD} pg_dump --host=${DB_HOST} --username=${DB_USER} --create --clean ${DB_OPTIONS} --dbname=${dbName} > /tmp/${dbName}.sql)
+    end=$(date +%s)
+
+    logger -p user.info "${dbName} backed up ($(stat -c %s /tmp/${dbName}.sql) bytes) in $(expr ${end} - ${start}) seconds."
+
+    runny gzip -f /tmp/${dbName}.sql
+    runny s3cmd put /tmp/${dbName}.sql.gz ${S3_BUCKET}
+#    runny aws s3 cp /tmp/${dbName}.sql.gz ${S3_BUCKET}
+
+    logger -p user.info "${dbName} backup stored in ${S3_BUCKET}."
+done

application/entrypoint.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env sh
+
+# hostname:port:database:username:password
+echo ${DB_HOST}:*:*:${DB_USER}:${DB_PASSWORD} > /root/.pgpass
+chmod 600 /root/.pgpass
+
+if [ "${LOGENTRIES_KEY}" ]; then
+    sed -i /etc/rsyslog.conf -e "s/LOGENTRIESKEY/${LOGENTRIES_KEY}/"
+    rsyslogd
+    sleep 10 # ensure rsyslogd is running before we may need to send logs to it
+else
+    logger -p user.error  "Missing LOGENTRIES_KEY environment variable"
+fi
+
+# default to every day at 2 am when no schedule is provided
+echo "${CRON_SCHEDULE:=0 2 * * *} runny /data/${MODE}.sh" >> /etc/crontabs/root
+
+runny $1

application/restore.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+
+for dbName in ${DB_NAMES}; do
+    logger -p user.info "restoring ${dbName}..."
+
+    runny s3cmd get -f ${S3_BUCKET}/${dbName}.sql.gz /tmp/${dbName}.sql.gz
+    runny gunzip -f /tmp/${dbName}.sql.gz
+
+    start=$(date +%s)
+    runny psql --host=${DB_HOST} --username=${DB_USER} ${DB_OPTIONS}  < /tmp/${dbName}.sql
+    end=$(date +%s)
+
+    logger -p user.info "${dbName} restored in $(expr ${end} - ${start}) seconds."
+done

docker-compose.yml

@@ -0,0 +1,56 @@
+version: '2'
+services:
+    data:
+        image: silintl/data-volume:latest
+        volumes:
+          - ./application:/data
+
+    # See https://hub.docker.com/_/postgres/ for details of the postgres image.
+    # POSTGRES_PASSWORD - superuser password for PostgreSQL
+    # POSTGRES_USER - superuser (default is 'postgres')
+    # POSTGRES_DB - name of default database (default is value of POSTGRES_USER)
+    db:
+        image: postgres:9.6-alpine
+        volumes_from:
+            - data
+        ports:
+            - "5432"
+        environment:
+            POSTGRES_PASSWORD: r00tp@ss!
+
+    adminer:
+        image: adminer:4.6.3
+        ports:
+            - "8080:8080"
+
+    # DB_HOST - hostname of the database server
+    # DB_USER - user that accesses the database
+    # DB_PASSWORD - password for the DB_USER
+    # DB_NAMES - list of databases to back up/restore
+    restore:
+        build: ./
+        volumes_from:
+            - data
+        env_file:
+            - ./local.env
+        environment:
+            DB_HOST: db
+            DB_USER: postgres
+            DB_PASSWORD: r00tp@ss!
+            DB_NAMES: world
+            MODE: restore
+            CRON_SCHEDULE: "25 * * * *"
+
+    backup:
+        build: ./
+        volumes_from:
+            - data
+        env_file:
+            - ./local.env
+        environment:
+            DB_HOST: db
+            DB_USER: postgres
+            DB_PASSWORD: r00tp@ss!
+            DB_NAMES: world
+            MODE: backup
+            CRON_SCHEDULE: "20 * * * *"

dockerbuild/rsyslog.conf

@@ -0,0 +1,16 @@
+# if you experience problems, check:
+# http://www.rsyslog.com/troubleshoot
+
+$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
+
+#
+# Configure TLS (logentries-specific example: https://docs.logentries.com/docs/rsyslog/)
+#
+$DefaultNetstreamDriverCAFile /etc/ssl/certs/ca-certificates.crt
+$ActionSendStreamDriver gtls
+$ActionSendStreamDriverMode 1
+$ActionSendStreamDriverAuthMode x509/name
+$ActionSendStreamDriverPermittedPeer *.logentries.com
+
+$template LogentriesFormat,"LOGENTRIESKEY %msg%\n"
+*.emerg,*.alert,*.crit,*.err,*.warning,user.* @@data.logentries.com:443;LogentriesFormat

local.env.dist

@@ -0,0 +1,4 @@
+LOGENTRIES_KEY=
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+S3_BUCKET=

test/README

@@ -0,0 +1,18 @@
+World
+=====
+
+World is a port of the example database available for MySQL on the mysql.com
+website.
+
+It is a simple 1:1 port and no attempt has been made to redesign the schema to
+better suit PostgreSQL.
+
+It is useful for comparison purposes and for learning how to execute simple
+queries and create simple tables, but is not particularly useful for exercising
+the advanced features of PostgreSQL or benchmarking.
+
+---
+(Taken from http://pgfoundry.org/projects/dbsamples/ on July 10, 2018.)
+
+The example database was loaded by hand and then backed up with the backup.sh
+script.  The resulting backup file was copied to this directory.