BackupDB 1.0

LICENSE

@@ -1,4 +1,6 @@
-Copyright (c) 2012, 2013, 2014, Fusionbox, Inc. All rights reserved.
+                              BSD 2-Clause License
+
+Copyright (c) 2012-2015, Fusionbox, Inc. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

MANIFEST.in

@@ -1,3 +1,2 @@
 include README.rst
 include LICENSE
-include test_settings.py

backupdb/backends/base.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import os
+import abc
+
+import spm
+import six
+
+
+class BaseBackend(six.with_metaclass(abc.ABCMeta, object)):
+    def __init__(self, **kwargs):
+        self.db_config = kwargs.pop('db_config')
+        self.backup_file = kwargs.pop('backup_file')
+        self.extra_args = kwargs.pop('extra_args', [])
+        self.show_ouput = kwargs.pop('show_ouput', False)
+
+    @abc.abstractmethod
+    def get_backup_command(self, db_name):
+        """
+        Returns a spm command which dumps the database on stdout
+        """
+
+    @abc.abstractmethod
+    def get_restore_command(self, db_name, drop_tables):
+        """
+        Returns a spm command which takes a backup file on stdin
+        """
+
+    def do_backup(self):
+        db_name = self.db_config['NAME']  # This has to raise KeyError if it does
+                                          # not exist.
+
+        command = self.get_backup_command(db_name)
+
+        proc = command.pipe('gzip')
+        proc.stdout = open(self.backup_file, 'w')
+        proc.wait()
+
+    def do_restore(self, drop_tables):
+        db_name = self.db_config['NAME']  # This has to raise KeyError if it does
+                                          # not exist.
+
+        if not os.path.isfile(self.backup_file):
+            raise RuntimeError("{} does not exist".format(fname))
+
+        command = self.get_restore_command(db_name, drop_tables=drop_tables)
+
+        proc = spm.run('zcat', self.backup_file).pipe(command)
+        proc.wait()

backupdb/backends/mysql.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import
+import os
+
+import spm
+
+from backupdb.utils import apply_arg_values
+from backupdb.backends.base import BaseBackend
+
+
+class MySQLBackend(BaseBackend):
+    extension = 'mysql'
+
+    def get_arguments(self):
+        return apply_arg_values(
+            ('--user={}', self.db_config.get('USER')),
+            ('--password={}', self.db_config.get('PASSWORD')),
+            ('--host={}', self.db_config.get('HOST')),
+            ('--port={}', self.db_config.get('PORT'))
+        )
+
+    def get_backup_command(self, db_name):
+        arguments = ['mysqldump'] + self.get_arguments() + self.extra_args + \
+                        [db_name]
+        return spm.run(*arguments)
+
+    def get_restore_command(self, db_name, drop_tables):
+        # TODO: Drop all tables from the database if drop_tables is True
+
+        arguments = ['mysql'] + self.get_arguments() + self.extra_args + \
+                        [db_name]
+        return spm.run(*arguments)

backupdb/backends/postgresql.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import
+import os
+
+import spm
+
+from backupdb.utils import apply_arg_values
+from backupdb.backends.base import BaseBackend
+
+# TODO: Use this command to drop tables from a database
+PG_DROP_SQL = """SELECT format('DROP TABLE IF EXISTS %I CASCADE;', tablename)
+                 FROM pg_tables WHERE schemaname = 'public';"""
+
+
+class PostgreSQLBackend(BaseBackend):
+    extension = 'pgsql'
+
+    def get_env(self):
+        try:
+            return {'PGPASSWORD': self.db_config['PASSWORD']}
+        except KeyError:
+            return {}
+
+    def get_arguments(self):
+        return apply_arg_values(
+            ('--username={0}', self.db_config.get('USER')),
+            ('--host={0}', self.db_config.get('HOST')),
+            ('--port={0}', self.db_config.get('PORT')),
+        )
+
+    def get_backup_command(self):
+        arguments = ['pg_dump', '--clean'] + self.get_arguments() + \
+                        self.extra_args + [database]
+        env = self.get_env()
+        return spm.run(*arguments, env=env)
+
+    def get_restore_command(self, drop_tables):
+        # TODO: Drop all tables from the database if drop_tables is True
+        database = self.db_config['NAME']
+
+        if not os.path.isfile(self.backup_file):
+            raise RuntimeError("{} does not exist".format(fname))
+
+        arguments = ['psql'] + self.get_arguments() + self.extra_args + \
+                        [database]
+        env = self.get_env()
+
+        return spm.run(*arguments, env=env)

backupdb/backends/sqlite.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import
+
+import spm
+
+from backupdb.backends.base import BaseBackend
+
+
+class SQLite3Backend(BaseBackend):
+    extension = 'sqlite'
+
+    def get_backup_command(self, fname):
+        arguments = ['sqlite3'] + self.extra_args + [fname, '.dump']
+        command = spm.run(*arguments)
+
+        return command
+
+    def get_restore_command(self, fname, drop_tables):
+        # TODO: Drop all tables from the database if drop_tables is True
+        return spm.run('sqlite3', fname)

backupdb/management/commands/backupdb.py

@@ -3,12 +3,13 @@
 from subprocess import CalledProcessError
 import logging
 import os
-import time
+from datetime import datetime
 
-from backupdb.utils.commands import BaseBackupDbCommand, do_postgresql_backup
-from backupdb.utils.exceptions import BackupError
-from backupdb.utils.log import section, SectionError, SectionWarning
-from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG
+from django.conf import settings
+from django.utils import timezone
+
+from backupdb.utils import BaseBackupDbCommand
+from backupdb.settings import get_backup_directory, BACKUP_CONFIG
 
 logger = logging.getLogger(__name__)
 
@@ -51,48 +52,54 @@ class Command(BaseBackupDbCommand):
     def handle(self, *args, **options):
         super(Command, self).handle(*args, **options)
 
-        from django.conf import settings
+        current_time_string = timezone.now().isoformat()
+
+        backup_suffix = options['backup_name']
+        if backup_suffix is None:
+            backup_suffix = current_time_string
 
-        current_time = time.strftime('%F-%s')
-        backup_name = options['backup_name'] or current_time
         show_output = options['show_output']
 
+        backup_directory = get_backup_directory()
+
         # Ensure backup dir present
-        if not os.path.exists(BACKUP_DIR):
-            os.makedirs(BACKUP_DIR)
+        if not os.path.exists(backup_directory):
+            os.makedirs(backup_directory)
 
         # Loop through databases
         for db_name, db_config in settings.DATABASES.items():
-            with section("Backing up '{0}'...".format(db_name)):
-                # Get backup config for this engine type
-                engine = db_config['ENGINE']
-                backup_config = BACKUP_CONFIG.get(engine)
-                if not backup_config:
-                    raise SectionWarning("Backup for '{0}' engine not implemented".format(engine))
-
-                # Get backup file name
-                backup_base_name = '{db_name}-{backup_name}.{backup_extension}.gz'.format(
-                    db_name=db_name,
-                    backup_name=backup_name,
-                    backup_extension=backup_config['backup_extension'],
-                )
-                backup_file = os.path.join(BACKUP_DIR, backup_base_name)
-
-                # Find backup command and get kwargs
-                backup_func = backup_config['backup_func']
-                backup_kwargs = {
-                    'backup_file': backup_file,
-                    'db_config': db_config,
-                    'show_output': show_output,
-                }
-                if backup_func is do_postgresql_backup:
-                    backup_kwargs['pg_dump_options'] = options['pg_dump_options']
-
-                # Run backup command
-                try:
-                    backup_func(**backup_kwargs)
-                    logger.info("Backup of '{db_name}' saved in '{backup_file}'".format(
+            logger.info("Starting to backup {!r}".format(db_name))
+            engine = db_config['ENGINE']
+            try:
+                backup_cls = BACKUP_CONFIG[engine]
+            except KeyError:
+                logger.error(
+                    "Backup for {!r} engine not implemented".format(engine))
+                continue  # TODO: Raise an error by default
+                          # TODO: implement --ignore-errors
+
+            fname = '{db}-{suffix}.{ext}.gz'.format(
+                db=db_name,
+                suffix=backup_suffix,
+                ext=backup_cls.extension,
+            )
+            absolute_fname = os.path.join(backup_directory, fname)
+
+            backup = backup_cls(
+                db_config=db_config,
+                backup_file=absolute_fname,
+                show_output=show_output,
+            )
+
+
+            try:
+                backup.do_backup()
+            except RuntimeError as e:
+                logger.error(e.message)
+            else:
+                logger.info(
+                    "Backup of {db_name!r} saved in {backup_file!r}".format(
                         db_name=db_name,
-                        backup_file=backup_file))
-                except (BackupError, CalledProcessError) as e:
-                    raise SectionError(e)
+                        backup_file=fname
+                    )
+                )

backupdb/management/commands/restoredb.py

@@ -8,11 +8,9 @@
 from django.conf import settings
 from django.db import close_connection
 
-from backupdb.utils.commands import BaseBackupDbCommand
-from backupdb.utils.exceptions import RestoreError
-from backupdb.utils.files import get_latest_timestamped_file
-from backupdb.utils.log import section, SectionError, SectionWarning
-from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG
+from backupdb.utils import get_latest_timestamped_file
+from backupdb.settings import get_backup_directory, BACKUP_CONFIG
+from backupdb.utils import BaseBackupDbCommand
 
 logger = logging.getLogger(__name__)
 
@@ -63,52 +61,57 @@ def handle(self, *args, **options):
         # Because of this psql can't drop django_content_types and just hangs
         close_connection()
 
+        backup_directory = get_backup_directory()
+
         # Ensure backup dir present
-        if not os.path.exists(BACKUP_DIR):
-            raise CommandError("Backup dir '{0}' does not exist!".format(BACKUP_DIR))
+        if not os.path.exists(backup_directory):
+            raise CommandError("Backup dir {!r} does not exist!".format(
+                backup_directory))
 
         backup_name = options['backup_name']
         drop_tables = options['drop_tables']
         show_output = options['show_output']
 
         # Loop through databases
         for db_name, db_config in settings.DATABASES.items():
-            with section("Restoring '{0}'...".format(db_name)):
-                # Get backup config for this engine type
-                engine = db_config['ENGINE']
-                backup_config = BACKUP_CONFIG.get(engine)
-                if not backup_config:
-                    raise SectionWarning("Restore for '{0}' engine not implemented".format(engine))
-
-                # Get backup file name
-                backup_extension = backup_config['backup_extension']
-                if backup_name:
-                    backup_file = '{dir}/{db_name}-{backup_name}.{ext}.gz'.format(
-                        dir=BACKUP_DIR,
-                        db_name=db_name,
-                        backup_name=backup_name,
-                        ext=backup_extension,
+            logger.info("Restoring {!r}".format(db_name))
+            engine = db_config['ENGINE']
+            try:
+                backup_cls = BACKUP_CONFIG[engine]
+            except KeyError:
+                logger.error(
+                    "Restore for {!r} engine not implemented".format(engine))
+                continue  # TODO: Raise an error by default
+                          # TODO: implement --ignore-errors
+
+
+
+            if backup_name:
+                fname = '{db}-{suffix}.{ext}.gz'.format(
+                    db=db_name,
+                    suffix=backup_suffix,
+                    ext=backup_cls.extension,
+                )
+            else:
+                fname = get_latest_timestamped_file(
+                    db_name, backup_cls.extension, backup_directory)
+                if fname is None:
+                    logger.error(
+                        "Couldn't find a default backup for '{!r}'".format(
+                            db_name)
                     )
-                else:
-                    try:
-                        backup_file = get_latest_timestamped_file(backup_extension)
-                    except RestoreError as e:
-                        raise SectionError(e)
-
-                # Find restore command and get kwargs
-                restore_func = backup_config['restore_func']
-                restore_kwargs = {
-                    'backup_file': backup_file,
-                    'db_config': db_config,
-                    'drop_tables': drop_tables,
-                    'show_output': show_output,
-                }
-
-                # Run restore command
-                try:
-                    restore_func(**restore_kwargs)
-                    logger.info("Restored '{db_name}' from '{backup_file}'".format(
-                        db_name=db_name,
-                        backup_file=backup_file))
-                except (RestoreError, CalledProcessError) as e:
-                    raise SectionError(e)
+                    continue  # TODO: Raise an error by default
+                              # TODO: implement --ignore-errors
+
+            absolute_fname = os.path.join(backup_directory, fname)
+
+            backup = backup_cls(
+                db_config=db_config,
+                backup_file=absolute_fname,
+                show_output=show_output,
+            )
+
+            backup.do_restore(drop_tables=drop_tables)
+            logger.info("Restored {db_name!r} from {backup_file!r}".format(
+                db_name=db_name,
+                backup_file=absolute_fname))