Merge pull request #192 from SUNET/jocar-mariadb-backup

Replicated backup node for MariaDB

lib/puppet/functions/ipv4_to_int.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+
+# Convert a single IPv4 address to an int
+
+require 'ipaddr'
+
+Puppet::Functions.create_function(:ipv4_to_int) do
+  def ipv4_to_int(*arguments)
+    IPAddr.new(arguments[0]).to_i
+  end
+end

manifests/mariadb.pp

@@ -4,8 +4,10 @@
   Integer $bootstrap=0,
   Array[Integer] $ports = [3306, 4444, 4567, 4568],
   Array[String] $dns = [],
+  Boolean $galera = true,
 )
 {
+
   $mariadb_root_password = lookup('mariadb_root_password', undef, undef,'NOT_SET_IN_HIERA')
   $mariadb_user = lookup('mariadb_user', undef, undef,undef)
   $mariadb_user_password = lookup('mariadb_user_password', undef, undef,undef)
@@ -14,7 +16,6 @@
   $clients = lookup('mariadb_clients', undef, undef,['127.0.0.1'])
   $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[])
   $mariadb_dir = '/opt/mariadb'
-  $server_id = 1000 + Integer($facts['networking']['hostname'][-1])
 
   # Hack to not clash with docker_compose which tries to create the same directory
   exec {'mariadb_dir_create':
@@ -54,26 +55,26 @@
     ok_criteria   => ['exit_status=0','max_age=2d'],
     warn_criteria => ['exit_status=1','max_age=3d'],
   }
-  file { '/usr/local/bin/cluster-size':
+  file { '/usr/local/bin/mariadb-galera-size':
     ensure  => present,
     content => template('sunet/mariadb/cluster-size.erb.sh'),
     mode    => '0744',
   }
-  file { '/usr/local/bin/cluster-status':
+  file { '/usr/local/bin/mariadb-galera-status':
     ensure  => present,
     content => template('sunet/mariadb/cluster-status.erb.sh'),
     mode    => '0744',
   }
-  file { '/etc/sudoers.d/99-size-test':
+  file { '/etc/sudoers.d/99-cluster-size-test':
     ensure  => file,
-    content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-size\n",
+    content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-size\n",
     mode    => '0440',
     owner   => 'root',
     group   => 'root',
   }
-  file { '/etc/sudoers.d/99-status-test':
+  file { '/etc/sudoers.d/99-cluster-status-test':
     ensure  => file,
-    content => "script ALL=(root) NOPASSWD: /usr/local/bin/cluster-status\n",
+    content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-galera-status\n",
     mode    => '0440',
     owner   => 'root',
     group   => 'root',
@@ -92,6 +93,8 @@
     content => template('sunet/mariadb/credentials.cnf.erb'),
     mode    => '0744',
   }
+
+  $server_id = ipv4_to_int($facts['networking']['ip'])
   file { "${mariadb_dir}/conf/my.cnf":
     ensure  => present,
     content => template('sunet/mariadb/my.cnf.erb'),

manifests/mariadb/backup.pp

@@ -0,0 +1,80 @@
+# This is a asyncronous replica of the Maria DB Cluster for SUNET
+class sunet::mariadb::backup(
+  String $mariadb_version=latest,
+  Array[String] $dns = [],
+  Boolean $backup_to_baas = true,
+  Boolean $nrpe = true,
+) {
+
+  sunet::mariadb { 'sunet_mariadb_simple':
+    mariadb_version => $mariadb_version,
+    ports           => [3306],
+    dns             => $dns,
+    galera          => false,
+  }
+
+  $cluster_nodes = lookup('mariadb_cluster_nodes', undef, undef,[])
+  $replicate_from = $cluster_nodes[0]
+
+  # Secrets from local.eyaml
+  $mariadb_root_password = safe_hiera('mariadb_root_password')
+  $mariadb_backup_password = safe_hiera('mariadb_root_password')
+  $mariadb_user_password = safe_hiera('mariadb_user_password')
+
+  file { '/opt/mariadb/scripts/start_replica_from_init.sh':
+    ensure  => present,
+    content => template('sunet/mariadb/backup/start_replica_from_init.erb.sh'),
+    mode    => '0744',
+  }
+  file { '/opt/mariadb/scripts/do_backup.sh':
+    ensure  => present,
+    content => template('sunet/mariadb/backup/do_backup.erb.sh'),
+    mode    => '0744',
+  }
+
+  if $backup_to_baas {
+    file { '/usr/local/bin/backup2baas':
+      ensure  => present,
+      content => template('sunet/mariadb/backup/backup2baas.erb'),
+      mode    => '0744',
+    }
+
+    sunet::scriptherder::cronjob { 'backup2baas':
+      cmd           => '/usr/local/bin/backup2baas',
+      hour          => '6',
+      minute        => '10',
+      ok_criteria   => ['exit_status=0', 'max_age=24h'],
+      warn_criteria => ['exit_status=1'],
+    }
+  }
+
+  file { '/usr/lib/nagios/plugins/check_mariadb-replication':
+    ensure  => present,
+    content => template('sunet/mariadb/backup/check_replication.erb'),
+    mode    => '0744',
+  }
+  file { '/usr/local/bin/mariadb-replication-status':
+    ensure  => present,
+    content => template('sunet/mariadb/backup/replication-status.erb'),
+    mode    => '0744',
+  }
+  file { '/etc/sudoers.d/99-mariadb-replication-test':
+    ensure  => file,
+    content => "script ALL=(root) NOPASSWD: /usr/local/bin/mariadb-replication-status",
+    mode    => '0440',
+    owner   => 'root',
+    group   => 'root',
+  }
+
+  if $nrpe {
+    sunet::sudoer {'nagios_run_replication_command':
+      user_name    => 'nagios',
+      collection   => 'nrpe_replication_check',
+      command_line => '/usr/lib/nagios/plugins/check_mariadb-replication'
+    }
+    sunet::nagios::nrpe_command {'check_async_replication':
+      command_line => '/usr/bin/sudo /usr/lib/nagios/plugins/check_mariadb-replication'
+    }
+  }
+
+}

templates/mariadb/backup/backup2baas.erb

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+docker exec mariadb-db-1 /scripts/do_backup.sh
+
+BACKUPDIR=/opt/mariadb/backups
+find "${BACKUPDIR}" -type f -mtime +31 -exec rm -f {} \;
+find "${BACKUPDIR}" -empty -type d -delete
+
+/usr/bin/dsmc backup

templates/mariadb/backup/check_replication.erb

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+result="$(/usr/local/bin/mariadb-replication-status)"
+if [[ "${result}" == "Slave_running	ON" ]]; then
+  echo "OK: Replica running"
+  exit 0
+else
+  echo "CRITICAL: Replica not running"
+  exit 2
+fi

templates/mariadb/backup/do_backup.erb.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+stream_name="mariadb-stream-$(date +%Y-%m-%dT%H.%M.%S).gz"
+dump_name="mariadb-dump-$(date +%Y-%m-%dT%H.%M.%S).sql.gz"
+backup_dir="/backups/$(date +%Y/%m/%d)"
+mkdir -p "${backup_dir}"
+
+buopts="--slave-info --safe-slave-backup"
+dumpopts="--dump-slave"
+mysql -p"${MYSQL_ROOT_PASSWORD}" -e "stop slave"
+mariadb-backup --backup ${buopts} -u root -p"${MYSQL_ROOT_PASSWORD}" --stream=xbstream | gzip >"${backup_dir}/${stream_name}"
+mysqldump --all-databases --single-transaction ${dumpopts} -u root -p${MYSQL_ROOT_PASSWORD} | gzip >"${backup_dir}/${dump_name}"
+mysql -p${MYSQL_ROOT_PASSWORD} -e "start slave"

templates/mariadb/backup/replication-status.erb

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec mariadb-db-1 mysql -u root -p'<%= @mariadb_root_password %>' -N -B -e "show status like 'Slave_running'"

templates/mariadb/backup/start_replica_from_init.erb.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+mysql="mysql -u root -p${MYSQL_ROOT_PASSWORD}"
+init_file='/backups/init.sql.gz'
+if [[ -f ${init_file} ]]; then
+	${mysql} -e "STOP SLAVE;RESET SLAVE;"
+	master_command=$(zgrep 'CHANGE MASTER TO MASTER_LOG_FILE' ${init_file} | sed -e 's/^-- //' -e 's/;$//')
+	master_command="${master_command}, MASTER_HOST='<%= @replicate_from %>', MASTER_USER='backup'"
+	master_command="${master_command}, MASTER_PASSWORD='<%= @mariadb_backup_password%>', MASTER_SSL=1"
+	master_command="${master_command}, MASTER_CONNECT_RETRY=20"
+	zcat ${init_file} | ${mysql}
+	${mysql} -e "${master_command}"
+	${mysql} -e "START SLAVE"
+	sleep 3s
+	${mysql} -e "SHOW SLAVE STATUS\G"
+fi
+
+exit 0

templates/mariadb/docker-compose_mariadb.yml.erb

@@ -11,7 +11,12 @@ services:
       - /opt/mariadb/datadir:/var/lib/mysql
       - /opt/mariadb/init:/docker-entrypoint-initdb.d
       - /opt/mariadb/scripts:/scripts
+<%- if @backup -%>
+      - /opt/mariadb_backup/start_replica_from_init.sh:/start_replica_from_init.sh
+<% end -%>
+<%- if @galera -%>
     network_mode: host
+<% end -%>
 <%- if [email protected]? -%>
     dns:
 <% @dns.each do |resolver| -%>
@@ -29,8 +34,10 @@ services:
 <%- if @mariadb_database -%>
       - MYSQL_DATABASE=<%= @mariadb_database %>
 <%- end -%>
+<%- if @galera -%>
       - BOOTSTRAP=<%= @bootstrap %>
       - FORCE_BOOTSTRAP=0
     command:  "--wsrep_cluster_address=gcomm://<%= @cluster_nodes.join(',') %>"
     tty: true
+<%- end -%>
 

templates/mariadb/my.cnf.erb

@@ -19,6 +19,8 @@ gtid_strict_mode        = ON
 log_bin                 = binlog
 log_slave_updates       = ON
 server_id               = <%= @server_id %>
+# Default hostname base relay_log is no good in containers
+relay_log               = 'relay-log'
 
 # Innodb
 innodb_autoinc_lock_mode        = 2
@@ -34,6 +36,7 @@ innodb_rollback_on_timeout      = 1
 innodb_write_io_threads         = 4    # CPU dependent
 transaction_isolation           = 'READ-COMMITTED'
 
+<% if @galera -%>
 # Galera
 wsrep_cluster_name              = "Sunet_MariaDB_Cluster"
 wsrep_gtid_domain_id            = 1000 # same on all Galera nodes in the same segment
@@ -45,3 +48,4 @@ wsrep_provider_options          = "gcache.size=2G;gmcast.segment=0" # gmcast.seg
 wsrep_slave_threads             = 4    # CPU dependent
 wsrep_sst_method                = mariabackup
 wsrep_sync_wait                 = 1
+<% end -%>