diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f9d69d --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Vagrant folder +.vagrant/ + +# phpstorm project files +.idea/ +/nbproject/ +# local config files +local.env + +vendor/ +*.aes diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fab22c4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine:3.6 + +RUN apk update \ + && apk add --no-cache rsyslog rsyslog-tls \ + ca-certificates openssl \ + bash \ + postgresql \ + postgresql-client \ + python py-pip \ + && update-ca-certificates \ + && pip install s3cmd python-magic + +COPY dockerbuild/rsyslog.conf /etc/rsyslog.conf + +RUN wget https://raw.githubusercontent.com/silinternational/runny/0.2/runny -O /usr/local/bin/runny \ + && chmod +x /usr/local/bin/runny + +COPY application/ /data/ +WORKDIR /data + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["crond -f"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2758cc6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 SIL International + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f9919f3 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +start: restore backup + +restore: db + docker-compose up -d restore + +backup: db + docker-compose up -d backup + +db: + docker-compose up -d db adminer + +clean: + docker-compose kill + docker system prune -f diff --git a/application/backup.sh b/application/backup.sh new file mode 100755 index 0000000..38773f8 --- /dev/null +++ b/application/backup.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +for dbName in ${DB_NAMES}; do + logger -p user.info "backing up ${dbName}..." + + start=$(date +%s) + runny $(PGPASSWORD=${DB_PASSWORD} pg_dump --host=${DB_HOST} --username=${DB_USER} --create --clean ${DB_OPTIONS} --dbname=${dbName} > /tmp/${dbName}.sql) + end=$(date +%s) + + logger -p user.info "${dbName} backed up ($(stat -c %s /tmp/${dbName}.sql) bytes) in $(expr ${end} - ${start}) seconds." + + runny gzip -f /tmp/${dbName}.sql + runny s3cmd put /tmp/${dbName}.sql.gz ${S3_BUCKET} +# runny aws s3 cp /tmp/${dbName}.sql.gz ${S3_BUCKET} + + logger -p user.info "${dbName} backup stored in ${S3_BUCKET}." +done diff --git a/application/entrypoint.sh b/application/entrypoint.sh new file mode 100755 index 0000000..abffb3f --- /dev/null +++ b/application/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +# hostname:port:database:username:password +echo ${DB_HOST}:*:*:${DB_USER}:${DB_PASSWORD} > /root/.pgpass +chmod 600 /root/.pgpass + +if [ "${LOGENTRIES_KEY}" ]; then + sed -i /etc/rsyslog.conf -e "s/LOGENTRIESKEY/${LOGENTRIES_KEY}/" + rsyslogd + sleep 10 # ensure rsyslogd is running before we may need to send logs to it +else + logger -p user.error "Missing LOGENTRIES_KEY environment variable" +fi + +# default to every day at 2 am when no schedule is provided +echo "${CRON_SCHEDULE:=0 2 * * *} runny /data/${MODE}.sh" >> /etc/crontabs/root + +runny $1 diff --git a/application/restore.sh b/application/restore.sh new file mode 100755 index 0000000..c52e35b --- /dev/null +++ b/application/restore.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +for dbName in ${DB_NAMES}; do + logger -p user.info "restoring ${dbName}..." + + runny s3cmd get -f ${S3_BUCKET}/${dbName}.sql.gz /tmp/${dbName}.sql.gz + runny gunzip -f /tmp/${dbName}.sql.gz + + start=$(date +%s) + runny psql --host=${DB_HOST} --username=${DB_USER} ${DB_OPTIONS} < /tmp/${dbName}.sql + end=$(date +%s) + + logger -p user.info "${dbName} restored in $(expr ${end} - ${start}) seconds." +done diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ab5cee4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '2' +services: + data: + image: silintl/data-volume:latest + volumes: + - ./application:/data + + # See https://hub.docker.com/_/postgres/ for details of the postgres image. + # POSTGRES_PASSWORD - superuser password for PostgreSQL + # POSTGRES_USER - superuser (default is 'postgres') + # POSTGRES_DB - name of default database (default is value of POSTGRES_USER) + db: + image: postgres:9.6-alpine + volumes_from: + - data + ports: + - "5432" + environment: + POSTGRES_PASSWORD: r00tp@ss! + + adminer: + image: adminer:4.6.3 + ports: + - "8080:8080" + + # DB_HOST - hostname of the database server + # DB_USER - user that accesses the database + # DB_PASSWORD - password for the DB_USER + # DB_NAMES - list of databases to back up/restore + restore: + build: ./ + volumes_from: + - data + env_file: + - ./local.env + environment: + DB_HOST: db + DB_USER: postgres + DB_PASSWORD: r00tp@ss! + DB_NAMES: world + MODE: restore + CRON_SCHEDULE: "25 * * * *" + + backup: + build: ./ + volumes_from: + - data + env_file: + - ./local.env + environment: + DB_HOST: db + DB_USER: postgres + DB_PASSWORD: r00tp@ss! + DB_NAMES: world + MODE: backup + CRON_SCHEDULE: "20 * * * *" diff --git a/dockerbuild/rsyslog.conf b/dockerbuild/rsyslog.conf new file mode 100644 index 0000000..2a2b661 --- /dev/null +++ b/dockerbuild/rsyslog.conf @@ -0,0 +1,16 @@ +# if you experience problems, check: +# http://www.rsyslog.com/troubleshoot + +$ModLoad imuxsock # provides support for local system logging (e.g. via logger command) + +# +# Configure TLS (logentries-specific example: https://docs.logentries.com/docs/rsyslog/) +# +$DefaultNetstreamDriverCAFile /etc/ssl/certs/ca-certificates.crt +$ActionSendStreamDriver gtls +$ActionSendStreamDriverMode 1 +$ActionSendStreamDriverAuthMode x509/name +$ActionSendStreamDriverPermittedPeer *.logentries.com + +$template LogentriesFormat,"LOGENTRIESKEY %msg%\n" +*.emerg,*.alert,*.crit,*.err,*.warning,user.* @@data.logentries.com:443;LogentriesFormat diff --git a/local.env.dist b/local.env.dist new file mode 100644 index 0000000..240dfe6 --- /dev/null +++ b/local.env.dist @@ -0,0 +1,4 @@ +LOGENTRIES_KEY= +AWS_ACCESS_KEY= +AWS_SECRET_KEY= +S3_BUCKET= diff --git a/test/README b/test/README new file mode 100644 index 0000000..8e16b7a --- /dev/null +++ b/test/README @@ -0,0 +1,18 @@ +World +===== + +World is a port of the example database available for MySQL on the mysql.com +website. + +It is a simple 1:1 port and no attempt has been made to redesign the schema to +better suit PostgreSQL. + +It is useful for comparison purposes and for learning how to execute simple +queries and create simple tables, but is not particularly useful for exercising +the advanced features of PostgreSQL or benchmarking. + +--- +(Taken from http://pgfoundry.org/projects/dbsamples/ on July 10, 2018.) + +The example database was loaded by hand and then backed up with the backup.sh +script. The resulting backup file was copied to this directory. diff --git a/test/world.sql.gz b/test/world.sql.gz new file mode 100644 index 0000000..0e7ca17 Binary files /dev/null and b/test/world.sql.gz differ