diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e0cc99 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docker-compose.override.yml \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8c791e0 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +language: node_js + +notifications: + email: false + +sudo: false + +node_js: + - "10" + +dist: trusty + +before_install: + - sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" + - sudo apt-get -qq update + - sudo apt-get install xz-utils + - export scversion="stable" + - wget "https://storage.googleapis.com/shellcheck/shellcheck-${scversion}.linux.x86_64.tar.xz" + - tar --xz -xvf shellcheck-"${scversion}".linux.x86_64.tar.xz + - sudo cp shellcheck-"${scversion}"/shellcheck /usr/bin/ + - shellcheck --version + - export ALLOW_EXTERNAL_SOURCE='-x ' + +env: + - CXX=g++-4.8 + +branches: + only: + - master + +script: + - ./dev/util/shellcheck.sh diff --git a/Dockerfile b/Dockerfile index 21b3288..40be80c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,35 @@ -FROM alpine:3.6 +ARG alpine_version +FROM alpine:${alpine_version} -ARG pg_alpine_branch -ARG pg_version +ARG alpine_version +ARG pg_full_version -# python for aws-cli, for s3 uploading, jq for dealing with AWS Cli -RUN apk --no-cache add python py-pip jq && \ - pip install awscli && \ - apk --purge -v del py-pip +#-------------------------------------------------------------------------------- +# Install dependencies +#-------------------------------------------------------------------------------- +# "postgresql" is required for "pg_restore" +# "py-pip" is required for "aws-cli" +#-------------------------------------------------------------------------------- +RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${alpine_version}/main" >> /etc/apk/repositories -# postgresql for pg_dump -RUN echo "http://dl-cdn.alpinelinux.org/alpine/${pg_alpine_branch}/main" >> /etc/apk/repositories -RUN echo ${pg_version}; apk --no-cache add postgresql=${pg_version} +RUN apk --no-cache --update add dumb-init postgresql=${pg_full_version} curl python3 && \ + curl https://bootstrap.pypa.io/get-pip.py | python3 && \ + pip install awscli && \ + rm -f /usr/bin/pip && \ + apk --purge -v del curl +#-------------------------------------------------------------------------------- +# Set script permissions and create required directories +#-------------------------------------------------------------------------------- COPY action.sh / RUN chmod +x action.sh +#-------------------------------------------------------------------------------- +# Use the `dumb-init` init system (PID 1) for process handling +#-------------------------------------------------------------------------------- +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +#-------------------------------------------------------------------------------- +# Configure and apply a cronjob +#-------------------------------------------------------------------------------- CMD echo "${CRON_MINUTE:-$(shuf -i 0-59 -n1)} ${CRON_HOUR:-*} * * * /action.sh" > /var/spool/cron/crontabs/root && crond -d 8 -f diff --git a/README.md b/README.md index bfd526b..e8ba44b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,25 @@ Cron based database dump and upload to S3. +## Build + +`./build_push.sh [-p , --package ]` + +`./build_push.sh -p 11.7-3.9` + +### Package files + +Each package file represents a release for a particular `postgres` branch. + +The contents of the latest package file may look like this: + +``` +ALPINE_VERSION='3.9' +PG_BASE_VERSION='11' +PG_FULL_VERSION='11.7' +PG_LATEST=true +``` + ## Usage Typically this image is instantiated as a container among many others and would have the responsibility of getting a database dump and uploading it to S3 at a particular time of day. diff --git a/build_push.sh b/build_push.sh index b528f39..581d5d5 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,23 +1,56 @@ -#! /bin/sh - -builds=$(echo ' -9.6.10 9.6.10-r0 v3.6 -#9.6.3 9.6.3-r0 v3.6 -9.6 9.6.10-r0 v3.6 -9 9.6.10-r0 v3.6 -latest 9.6.10-r0 v3.6 -#9.5.7 9.5.7-r0 v3.4 -#9.5 9.5.7-r0 v3.4 -' | grep -v '^#' | tr -s ' ') - -IFS=$'\n' -for build in $builds; do - tag=$(echo $build | cut -d ' ' -f 1 ) - pgVersion=$(echo $build | cut -d ' ' -f 2) - pgAlpineBranch=$(echo $build | cut -d ' ' -f 3) - - echo docker build --tag bluedrop360/postgres-dump-to-s3:$tag --build-arg pg_version="$pgVersion" --build-arg pg_alpine_branch=$pgAlpineBranch . - eval docker build --tag bluedrop360/postgres-dump-to-s3:$tag --build-arg pg_version="$pgVersion" --build-arg pg_alpine_branch=$pgAlpineBranch . - echo docker push bluedrop360/postgres-dump-to-s3:$tag - eval docker push bluedrop360/postgres-dump-to-s3:$tag +#!/usr/bin/env sh + +#------------------------------------------------------------------------------------ +# Loop over arguments +#------------------------------------------------------------------------------------ +for arg in "$@"; do + # [ -p | --package ] + if [ -n "${package_flag}" ]; then + package_flag='' + package="${arg}" + fi + + if [ "${arg}" = "-p" ] || [ "${arg}" = "--package" ]; then + package_flag=true + fi +done + +#------------------------------------------------------------------------------------ +# Exit on error +#------------------------------------------------------------------------------------ +if [ -z "${package}" ]; then + echo '> Package file not specified: [-p , --package ]' + exit 127 +fi + +if [ -f "./package/${package}.env" ]; then + . "./package/${package}.env" +else + echo "> Package file not found: './package/${package}.env'" + exit 127 +fi + +builds=\ +"${PG_FULL_VERSION}_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}",\ +"${PG_BASE_VERSION}_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}" + +if [ "${PG_LATEST:-'false'}" = 'true' ]; then + builds="${builds}","latest_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}" +fi + +echo $builds | tr ',' '\n' | while read build; do + ALPINE_VERSION=$(echo "${build}" | cut -d '_' -f 3) + pg_dump_to_s3_tag=$(echo "${build}" | cut -d '_' -f 1 ) + PG_FULL_VERSION=$(echo "${build}" | cut -d '_' -f 2) + + echo "" + echo "--------------------------------" + echo "POSTGRES-DUMP-TO-S3 TAG: ${pg_dump_to_s3_tag}" + echo "POSTGRES PACKAGE VERSION: ${PG_FULL_VERSION}" + echo "--------------------------------" + + echo docker build --tag bluedrop360/postgres-dump-to-s3:$pg_dump_to_s3_tag --build-arg pg_full_version=$PG_FULL_VERSION --build-arg alpine_version="${ALPINE_VERSION}" . + eval docker build --tag bluedrop360/postgres-dump-to-s3:$pg_dump_to_s3_tag --build-arg pg_full_version=$PG_FULL_VERSION --build-arg alpine_version="${ALPINE_VERSION}" . || exit 1 + echo docker push bluedrop360/postgres-dump-to-s3:$pg_dump_to_s3_tag + eval docker push bluedrop360/postgres-dump-to-s3:$pg_dump_to_s3_tag || exit 1 done diff --git a/dev/util/shellcheck.sh b/dev/util/shellcheck.sh new file mode 100755 index 0000000..af7e20d --- /dev/null +++ b/dev/util/shellcheck.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +printf '\n%s\n' "Checking shell scripts..." + +SHELLCHECK_OPTS="" + +RUN_SHELLCHECK="shellcheck ${ALLOW_EXTERNAL_SOURCE:-} ${SHELLCHECK_OPTS} {} +" +eval "find ./*.sh -type f -exec ${RUN_SHELLCHECK}" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..744e16f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +######################################################## +# Docker Compose: https://docs.docker.com/compose +######################################################## +# This file is for local testing +# - Copy contents to `docker-compose.override.yml` +# - Update the volume and environment variable values +# - BUILD: docker-compose build +# - BUILD AND RUN: docker-compose up --build +######################################################## +version: '3' + +services: + postgres-restore-from-s3: + image: postgres-restore-from-s3:11.7 + network_mode: 'host' + build: + context: ./ + dockerfile: ./Dockerfile + args: + alpine_version: '3.9' + pg_full_version: '11.7-r0' + environment: + AWS_BUCKET: + AWS_REGION: + DATABASE_URL: postgres://:@:/ + DUMP_OBJECT_PREFIX: /postgres/ + volumes: + - /build/postgres/dump:/cache:rw diff --git a/package/10.12-3.8.env b/package/10.12-3.8.env new file mode 100644 index 0000000..c1f48eb --- /dev/null +++ b/package/10.12-3.8.env @@ -0,0 +1,3 @@ +ALPINE_VERSION='3.8' +PG_BASE_VERSION='10' +PG_FULL_VERSION='10.12' diff --git a/package/11.7-3.9.env b/package/11.7-3.9.env new file mode 100644 index 0000000..970ec5b --- /dev/null +++ b/package/11.7-3.9.env @@ -0,0 +1,4 @@ +ALPINE_VERSION='3.9' +PG_LATEST=false +PG_BASE_VERSION='11' +PG_FULL_VERSION='11.7' diff --git a/package/9.6.13-3.6.env b/package/9.6.13-3.6.env new file mode 100644 index 0000000..9ee2ce9 --- /dev/null +++ b/package/9.6.13-3.6.env @@ -0,0 +1,3 @@ +ALPINE_VERSION='3.6' +PG_BASE_VERSION='9' +PG_FULL_VERSION='9.6.13'