implement ZFS-backed rsnapshot backup

hieradata/dummy_secrets.yaml

@@ -15,6 +15,7 @@ ocfbackups::box:
   api_client_id: dummy_client_id
   api_client_secret: dummy_client_secret
 ocfbackups::mysql::password: dummypassword
+ocfbackups::offsite_host: dummyhost
 
 sensu::redis::password: dummypassword
 

modules/ocf_backups/files/backup-mysql

@@ -18,4 +18,4 @@ parallel -i \
              --triggers \
              --routines \
              --single-transaction \
-             --databases {} | pigz > "mysql-{}-$(date +%F).sql.gz"' -- $databases
+             --databases {} > "mysql-{}-$(date +%F).sql"' -- $databases

modules/ocf_backups/files/backup-pgsql

@@ -3,4 +3,4 @@ set -euo pipefail
 
 # Dumps the entire PostgreSQL instance to one .sql file.
 # Requires that a valid ~/.pgpass file be available on the PostgreSQL host
-ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost | pigz' > "pgsql-all-$(date +%F).sql.gz"
+ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost' > "pgsql-all-$(date +%F).sql"

modules/ocf_backups/files/backup-zfs-logrotate

@@ -0,0 +1,5 @@
+/var/log/ocf-backup-zfs.log {
+	rotate 100
+	daily
+	compress
+}

modules/ocf_backups/files/backup-zfs.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+CURRENT_SNAPSHOT_FILE=/opt/share/backups/current-zfs-snapshot
+CURRENT_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE)
+OFFSITE_HOST=$(cat /opt/share/backups/offsite-host)
+echo "$CURRENT_SNAPSHOT"
+
+rsnapshot -c /opt/share/backups/rsnapshot-zfs.conf sync
+rsnapshot -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync
+rsnapshot -c /opt/share/backups/rsnapshot-zfs-git.conf sync
+rsnapshot -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync
+
+zfs-auto-snapshot --syslog --label=after-backup --keep=10 // | awk -F"," '{print $1}' | cut -c2- > $CURRENT_SNAPSHOT_FILE
+NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE)
+
+echo "$CURRENT_SNAPSHOT"
+echo "$NEW_SNAPSHOT"
+
+syncoid -r --no-sync-snap --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot

modules/ocf_backups/files/rsnapshot-zfs-git.conf

@@ -0,0 +1,45 @@
+#################################################
+# rsnapshot.conf - rsnapshot configuration file #
+#################################################
+#                                               #
+# PLEASE BE AWARE OF THE FOLLOWING RULES:       #
+#                                               #
+# This file requires tabs between elements      #
+#                                               #
+# Directories require a trailing slash:         #
+#   right: /home/                               #
+#   wrong: /home                                #
+#                                               #
+#################################################
+
+config_version	1.2
+
+cmd_cp	/bin/cp
+cmd_rm	/bin/rm
+cmd_rsync	/usr/local/bin/rsync-no-vanished
+cmd_ssh	/usr/bin/ssh
+cmd_logger	/usr/bin/logger
+
+# remote backups require login as ocfbackups, then `sudo rsync-no-vanished'
+cmd_preexec	/usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups
+cmd_postexec	/usr/bin/kdestroy
+
+# default is "--delete --numeric-ids --relative --delete-excluded"
+# we add the 'sudo rsync-no-vanished' bits
+rsync_long_args	--delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished"
+
+no_create_root	1
+one_fs	1
+sync_first	1
+
+lockfile	/run/rsnapshot.pid
+
+# backup root directory
+snapshot_root	/backup/encrypted/rsnapshot/git/
+
+retain	daily	1
+# backup points/scripts
+# nfs (homedirs, webdirs)
+
+# scripts
+backup_script	/opt/share/backups/backup-git	.

modules/ocf_backups/files/rsnapshot-zfs-mysql.conf

@@ -0,0 +1,45 @@
+#################################################
+# rsnapshot.conf - rsnapshot configuration file #
+#################################################
+#                                               #
+# PLEASE BE AWARE OF THE FOLLOWING RULES:       #
+#                                               #
+# This file requires tabs between elements      #
+#                                               #
+# Directories require a trailing slash:         #
+#   right: /home/                               #
+#   wrong: /home                                #
+#                                               #
+#################################################
+
+config_version	1.2
+
+cmd_cp	/bin/cp
+cmd_rm	/bin/rm
+cmd_rsync	/usr/local/bin/rsync-no-vanished
+cmd_ssh	/usr/bin/ssh
+cmd_logger	/usr/bin/logger
+
+# remote backups require login as ocfbackups, then `sudo rsync-no-vanished'
+cmd_preexec	/usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups
+cmd_postexec	/usr/bin/kdestroy
+
+# default is "--delete --numeric-ids --relative --delete-excluded"
+# we add the 'sudo rsync-no-vanished' bits
+rsync_long_args	--delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished"
+
+no_create_root	1
+one_fs	1
+sync_first	1
+
+lockfile	/run/rsnapshot.pid
+
+# backup root directory
+snapshot_root	/backup/encrypted/rsnapshot/mysql/
+
+retain	daily	1
+# backup points/scripts
+# nfs (homedirs, webdirs)
+
+# scripts
+backup_script	/opt/share/backups/backup-mysql	.

modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf

@@ -0,0 +1,45 @@
+#################################################
+# rsnapshot.conf - rsnapshot configuration file #
+#################################################
+#                                               #
+# PLEASE BE AWARE OF THE FOLLOWING RULES:       #
+#                                               #
+# This file requires tabs between elements      #
+#                                               #
+# Directories require a trailing slash:         #
+#   right: /home/                               #
+#   wrong: /home                                #
+#                                               #
+#################################################
+
+config_version	1.2
+
+cmd_cp	/bin/cp
+cmd_rm	/bin/rm
+cmd_rsync	/usr/local/bin/rsync-no-vanished
+cmd_ssh	/usr/bin/ssh
+cmd_logger	/usr/bin/logger
+
+# remote backups require login as ocfbackups, then `sudo rsync-no-vanished'
+cmd_preexec	/usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups
+cmd_postexec	/usr/bin/kdestroy
+
+# default is "--delete --numeric-ids --relative --delete-excluded"
+# we add the 'sudo rsync-no-vanished' bits
+rsync_long_args	--delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished"
+
+no_create_root	1
+one_fs	1
+sync_first	1
+
+lockfile	/run/rsnapshot.pid
+
+# backup root directory
+snapshot_root	/backup/encrypted/rsnapshot/pgsql/
+
+retain	daily	1
+# backup points/scripts
+# nfs (homedirs, webdirs)
+
+# scripts
+backup_script	/opt/share/backups/backup-pgsql	.

modules/ocf_backups/files/rsnapshot-zfs.conf

@@ -0,0 +1,74 @@
+#################################################
+# rsnapshot.conf - rsnapshot configuration file #
+#################################################
+#                                               #
+# PLEASE BE AWARE OF THE FOLLOWING RULES:       #
+#                                               #
+# This file requires tabs between elements      #
+#                                               #
+# Directories require a trailing slash:         #
+#   right: /home/                               #
+#   wrong: /home                                #
+#                                               #
+#################################################
+
+config_version	1.2
+
+cmd_cp	/bin/cp
+cmd_rm	/bin/rm
+cmd_rsync	/usr/local/bin/rsync-no-vanished
+cmd_ssh	/usr/bin/ssh
+cmd_logger	/usr/bin/logger
+
+# remote backups require login as ocfbackups, then `sudo rsync-no-vanished'
+cmd_preexec	/usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups
+cmd_postexec	/usr/bin/kdestroy
+
+# default is "--delete --numeric-ids --relative --delete-excluded"
+# we add the 'sudo rsync-no-vanished' bits
+rsync_long_args	 --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished"
+
+no_create_root	1
+one_fs	1
+sync_first	1
+
+lockfile	/run/rsnapshot.pid
+
+# backup root directory
+snapshot_root	/backup/encrypted/rsnapshot/
+
+retain	daily	1
+# backup points/scripts
+# scripts
+# nfs (homedirs, webdirs)
+backup	ocfbackups@filehost:/opt/homes/	nfs/
+
+# remote servers
+backup	ocfbackups@hal:/etc/libvirt/qemu/	servers/vm_xml/hal/
+backup	ocfbackups@jaws:/etc/libvirt/qemu/	servers/vm_xml/jaws/
+backup	ocfbackups@pandemic:/etc/libvirt/qemu/	servers/vm_xml/pandemic/
+backup	ocfbackups@riptide:/etc/libvirt/qemu/	servers/vm_xml/riptide/
+backup	ocfbackups@scurvy:/etc/libvirt/qemu/	servers/vm_xml/scurvy/
+backup	ocfbackups@kerberos:/var/lib/heimdal-kdc/	servers/kerberos/
+backup	ocfbackups@kerberos:/var/backups/kerberos/	servers/kerberos/
+backup	ocfbackups@ldap:/var/lib/ldap/	servers/ldap/
+backup	ocfbackups@ldap:/var/backups/ldap/	servers/ldap/
+
+backup	ocfbackups@puppet:/etc/puppetlabs/	servers/puppet/
+backup	ocfbackups@puppet:/opt/puppetlabs/	servers/puppet/
+
+backup	ocfbackups@puppetdb:/etc/puppetlabs/puppet/ssl/	servers/puppetdb/
+
+backup	ocfbackups@munin:/var/lib/munin/	servers/munin/
+
+backup	ocfbackups@apt:/opt/apt/	servers/apt/
+
+backup	ocfbackups@jenkins:/var/lib/jenkins/	servers/jenkins/
+
+backup	ocfbackups@rancid:/var/lib/rancid/	servers/rancid/
+
+backup	ocfbackups@ns:/etc/bind/keys/	servers/ns/
+
+backup	ocfbackups@irc:/var/lib/znc/	servers/irc/
+
+# vim: ts=16 sts=16 sw=16 noet

modules/ocf_backups/manifests/init.pp

@@ -13,6 +13,15 @@
       ensure => directory,
       group  => ocfroot,
       mode   => '0750';
+
+    '/opt/share/backups/offsite-host':
+        content => lookup('ocfbackups::offsite_host'),
+        owner   => root,
+        group   => root,
+        mode    => '0400';
+
+    '/etc/logrotate.d/backup-zfs':
+      source  => 'puppet:///modules/ocf_backups/backup-zfs-logrotate';
   }
 
   # keytab for ocfbackups user, used to rsync from remote servers

modules/ocf_backups/manifests/rsnapshot.pp

@@ -5,6 +5,21 @@
     '/opt/share/backups/rsnapshot.conf':
       source => 'puppet:///modules/ocf_backups/rsnapshot.conf';
 
+    '/opt/share/backups/rsnapshot-zfs.conf':
+      source => 'puppet:///modules/ocf_backups/rsnapshot-zfs.conf';
+
+    '/opt/share/backups/rsnapshot-zfs-mysql.conf':
+      source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-mysql.conf';
+    '/opt/share/backups/rsnapshot-zfs-pgsql.conf':
+      source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-pgsql.conf';
+    '/opt/share/backups/rsnapshot-zfs-git.conf':
+      source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-git.conf';
+
+    '/usr/local/sbin/backup-zfs.sh':
+      source => 'puppet:///modules/ocf_backups/backup-zfs.sh',
+      mode   => '0755';
+
+    # TODO: update for ZFS
     '/opt/share/backups/check-rsnapshot-backups':
       source => 'puppet:///modules/ocf_backups/check-rsnapshot-backups',
       mode   => '0755';
@@ -13,48 +28,17 @@
 
   # TODO: update times listed here after move to remote backups
 
-  # Since we use sync_first, actual backups only happen at the most frequent
-  # ("smallest") backup level, i.e. daily.
-  #
-  # The other backup levels just promote a daily backup into a weekly/monthly
-  # one, so they are comparatively fast.
-  #
-  # As of 2015-03-29, it takes 30 minutes to do a promotion, and 4 hours to do
-  # a full backup. So we leave 2 hours for promotions and 8 hours for a full
-  # backup to be safe.
-  #
-  # It's important that jobs don't overlap, so our plan is:
-  #     10pm-12am monthly backup takes place (~30 minutes)
-  #     12am-2am: weekly backup takes place (~30 minutes)
-  #     2am-10am: daily backup takes place (~4 hours)
-
-  $rsnapshot = 'rsnapshot -c /opt/share/backups/rsnapshot.conf'
+  $rsnapshot = '/usr/local/sbin/backup-zfs.sh | tee -a /var/log/zfs-backup.log'
 
   cron {
     default:
       user   => root,
       minute => '0';
 
-    # 10pm on 1st of month
-    'rsnapshot-monthly':
-      command  => "${rsnapshot} monthly",
-      hour     => '22',
-      monthday => '1';
-
-    # 12am Saturday mornings
-    'rsnapshot-weekly':
-      command => "${rsnapshot} weekly",
-      hour    => '0',
-      weekday => '6';
-
-    # 2am daily
+    # ZFS
     'rsnapshot-daily':
-      command => "${rsnapshot} sync && ${rsnapshot} daily",
-      hour    => '2';
-
-    # check rsnapshot backups to ensure they're actually happening
-    'check-rsnapshot-backups':
-      command => '/opt/share/backups/check-rsnapshot-backups',
-      hour    => '10';
+      command => $rsnapshot,
+      hour    => '03',
+      minute  => '00';
   }
 }