diff --git a/hieradata/dummy_secrets.yaml b/hieradata/dummy_secrets.yaml index 8ae8406cf..9b6efca5a 100644 --- a/hieradata/dummy_secrets.yaml +++ b/hieradata/dummy_secrets.yaml @@ -15,6 +15,7 @@ ocfbackups::box: api_client_id: dummy_client_id api_client_secret: dummy_client_secret ocfbackups::mysql::password: dummypassword +ocfbackups::offsite_host: dummyhost sensu::redis::password: dummypassword diff --git a/modules/ocf_backups/files/backup-mysql b/modules/ocf_backups/files/backup-mysql index 132629da0..024d7ff39 100755 --- a/modules/ocf_backups/files/backup-mysql +++ b/modules/ocf_backups/files/backup-mysql @@ -18,4 +18,4 @@ parallel -i \ --triggers \ --routines \ --single-transaction \ - --databases {} | pigz > "mysql-{}-$(date +%F).sql.gz"' -- $databases + --databases {} > "mysql-{}-$(date +%F).sql"' -- $databases diff --git a/modules/ocf_backups/files/backup-pgsql b/modules/ocf_backups/files/backup-pgsql index 6fd361f01..0f76953fc 100755 --- a/modules/ocf_backups/files/backup-pgsql +++ b/modules/ocf_backups/files/backup-pgsql @@ -3,4 +3,4 @@ set -euo pipefail # Dumps the entire PostgreSQL instance to one .sql file. # Requires that a valid ~/.pgpass file be available on the PostgreSQL host -ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost | pigz' > "pgsql-all-$(date +%F).sql.gz" +ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost' > "pgsql-all-$(date +%F).sql" diff --git a/modules/ocf_backups/files/backup-zfs-logrotate b/modules/ocf_backups/files/backup-zfs-logrotate new file mode 100644 index 000000000..b549cc46c --- /dev/null +++ b/modules/ocf_backups/files/backup-zfs-logrotate @@ -0,0 +1,5 @@ +/var/log/ocf-backup-zfs.log { + rotate 100 + daily + compress +} diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh new file mode 100755 index 000000000..09a813740 --- /dev/null +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +CURRENT_SNAPSHOT_FILE=/opt/share/backups/current-zfs-snapshot +CURRENT_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) +OFFSITE_HOST=$(cat /opt/share/backups/offsite-host) +echo "$CURRENT_SNAPSHOT" + +rsnapshot -c /opt/share/backups/rsnapshot-zfs.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-git.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync + +zfs-auto-snapshot --syslog --label=after-backup --keep=10 // | awk -F"," '{print $1}' | cut -c2- > $CURRENT_SNAPSHOT_FILE +NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) + +echo "$CURRENT_SNAPSHOT" +echo "$NEW_SNAPSHOT" + +syncoid -r --no-sync-snap --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot diff --git a/modules/ocf_backups/files/rsnapshot-zfs-git.conf b/modules/ocf_backups/files/rsnapshot-zfs-git.conf new file mode 100644 index 000000000..1d3617473 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-git.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/git/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-git . diff --git a/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf b/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf new file mode 100644 index 000000000..04124bbd9 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/mysql/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-mysql . diff --git a/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf b/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf new file mode 100644 index 000000000..3cdec78d5 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/pgsql/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-pgsql . diff --git a/modules/ocf_backups/files/rsnapshot-zfs.conf b/modules/ocf_backups/files/rsnapshot-zfs.conf new file mode 100644 index 000000000..166ff3f3b --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs.conf @@ -0,0 +1,74 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/ + +retain daily 1 +# backup points/scripts +# scripts +# nfs (homedirs, webdirs) +backup ocfbackups@filehost:/opt/homes/ nfs/ + +# remote servers +backup ocfbackups@hal:/etc/libvirt/qemu/ servers/vm_xml/hal/ +backup ocfbackups@jaws:/etc/libvirt/qemu/ servers/vm_xml/jaws/ +backup ocfbackups@pandemic:/etc/libvirt/qemu/ servers/vm_xml/pandemic/ +backup ocfbackups@riptide:/etc/libvirt/qemu/ servers/vm_xml/riptide/ +backup ocfbackups@scurvy:/etc/libvirt/qemu/ servers/vm_xml/scurvy/ +backup ocfbackups@kerberos:/var/lib/heimdal-kdc/ servers/kerberos/ +backup ocfbackups@kerberos:/var/backups/kerberos/ servers/kerberos/ +backup ocfbackups@ldap:/var/lib/ldap/ servers/ldap/ +backup ocfbackups@ldap:/var/backups/ldap/ servers/ldap/ + +backup ocfbackups@puppet:/etc/puppetlabs/ servers/puppet/ +backup ocfbackups@puppet:/opt/puppetlabs/ servers/puppet/ + +backup ocfbackups@puppetdb:/etc/puppetlabs/puppet/ssl/ servers/puppetdb/ + +backup ocfbackups@munin:/var/lib/munin/ servers/munin/ + +backup ocfbackups@apt:/opt/apt/ servers/apt/ + +backup ocfbackups@jenkins:/var/lib/jenkins/ servers/jenkins/ + +backup ocfbackups@rancid:/var/lib/rancid/ servers/rancid/ + +backup ocfbackups@ns:/etc/bind/keys/ servers/ns/ + +backup ocfbackups@irc:/var/lib/znc/ servers/irc/ + +# vim: ts=16 sts=16 sw=16 noet diff --git a/modules/ocf_backups/manifests/init.pp b/modules/ocf_backups/manifests/init.pp index 57ff99b0b..fc9630db9 100644 --- a/modules/ocf_backups/manifests/init.pp +++ b/modules/ocf_backups/manifests/init.pp @@ -13,6 +13,15 @@ ensure => directory, group => ocfroot, mode => '0750'; + + '/opt/share/backups/offsite-host': + content => lookup('ocfbackups::offsite_host'), + owner => root, + group => root, + mode => '0400'; + + '/etc/logrotate.d/backup-zfs': + source => 'puppet:///modules/ocf_backups/backup-zfs-logrotate'; } # keytab for ocfbackups user, used to rsync from remote servers diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 8af41eb58..c4330776d 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -5,6 +5,21 @@ '/opt/share/backups/rsnapshot.conf': source => 'puppet:///modules/ocf_backups/rsnapshot.conf'; + '/opt/share/backups/rsnapshot-zfs.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs.conf'; + + '/opt/share/backups/rsnapshot-zfs-mysql.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-mysql.conf'; + '/opt/share/backups/rsnapshot-zfs-pgsql.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-pgsql.conf'; + '/opt/share/backups/rsnapshot-zfs-git.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-git.conf'; + + '/usr/local/sbin/backup-zfs.sh': + source => 'puppet:///modules/ocf_backups/backup-zfs.sh', + mode => '0755'; + + # TODO: update for ZFS '/opt/share/backups/check-rsnapshot-backups': source => 'puppet:///modules/ocf_backups/check-rsnapshot-backups', mode => '0755'; @@ -13,48 +28,17 @@ # TODO: update times listed here after move to remote backups - # Since we use sync_first, actual backups only happen at the most frequent - # ("smallest") backup level, i.e. daily. - # - # The other backup levels just promote a daily backup into a weekly/monthly - # one, so they are comparatively fast. - # - # As of 2015-03-29, it takes 30 minutes to do a promotion, and 4 hours to do - # a full backup. So we leave 2 hours for promotions and 8 hours for a full - # backup to be safe. - # - # It's important that jobs don't overlap, so our plan is: - # 10pm-12am monthly backup takes place (~30 minutes) - # 12am-2am: weekly backup takes place (~30 minutes) - # 2am-10am: daily backup takes place (~4 hours) - - $rsnapshot = 'rsnapshot -c /opt/share/backups/rsnapshot.conf' + $rsnapshot = '/usr/local/sbin/backup-zfs.sh | tee -a /var/log/zfs-backup.log' cron { default: user => root, minute => '0'; - # 10pm on 1st of month - 'rsnapshot-monthly': - command => "${rsnapshot} monthly", - hour => '22', - monthday => '1'; - - # 12am Saturday mornings - 'rsnapshot-weekly': - command => "${rsnapshot} weekly", - hour => '0', - weekday => '6'; - - # 2am daily + # ZFS 'rsnapshot-daily': - command => "${rsnapshot} sync && ${rsnapshot} daily", - hour => '2'; - - # check rsnapshot backups to ensure they're actually happening - 'check-rsnapshot-backups': - command => '/opt/share/backups/check-rsnapshot-backups', - hour => '10'; + command => $rsnapshot, + hour => '03', + minute => '00'; } }