From 6f25cf12047ebf67e69c9897c9562ce302f2714a Mon Sep 17 00:00:00 2001 From: Antoine Catton Date: Mon, 6 Apr 2015 18:10:51 -0600 Subject: [PATCH 1/3] Huge refactoring using polymorphism, and removing useless code --- LICENSE | 4 +- MANIFEST.in | 1 - backupdb/{utils => backends}/__init__.py | 0 backupdb/backends/base.py | 48 +++ backupdb/backends/mysql.py | 31 ++ backupdb/backends/postgresql.py | 47 +++ backupdb/backends/sqlite.py | 19 + .../app/models.py => backends/sqlite3.py} | 0 backupdb/management/commands/backupdb.py | 91 +++-- backupdb/management/commands/restoredb.py | 93 ++--- backupdb/settings.py | 19 + backupdb/tests/__init__.py | 21 - backupdb/tests/app/urls.py | 2 - backupdb/tests/commands.py | 367 ------------------ backupdb/tests/files.py | 37 -- backupdb/tests/log.py | 47 --- backupdb/tests/processes.py | 170 -------- backupdb/tests/utils.py | 43 -- backupdb/utils.py | 65 ++++ backupdb/utils/commands.py | 163 -------- backupdb/utils/exceptions.py | 6 - backupdb/utils/files.py | 25 -- backupdb/utils/log.py | 67 ---- backupdb/utils/processes.py | 96 ----- backupdb/utils/settings.py | 36 -- setup.py | 19 +- test_settings.py | 23 -- tests/integration_test.py | 47 +++ tests/unit_test.py | 82 ++++ unit_tests_scratch/.gitkeep | 0 30 files changed, 469 insertions(+), 1200 deletions(-) rename backupdb/{utils => backends}/__init__.py (100%) create mode 100644 backupdb/backends/base.py create mode 100644 backupdb/backends/mysql.py create mode 100644 backupdb/backends/postgresql.py create mode 100644 backupdb/backends/sqlite.py rename backupdb/{tests/app/models.py => backends/sqlite3.py} (100%) create mode 100644 backupdb/settings.py delete mode 100644 backupdb/tests/__init__.py delete mode 100644 backupdb/tests/app/urls.py delete mode 100644 backupdb/tests/commands.py delete mode 100644 backupdb/tests/files.py delete mode 100644 backupdb/tests/log.py delete mode 100644 backupdb/tests/processes.py delete mode 100644 backupdb/tests/utils.py create mode 100644 backupdb/utils.py delete mode 100644 backupdb/utils/commands.py delete mode 100644 backupdb/utils/exceptions.py delete mode 100644 backupdb/utils/files.py delete mode 100644 backupdb/utils/log.py delete mode 100644 backupdb/utils/processes.py delete mode 100644 backupdb/utils/settings.py delete mode 100644 test_settings.py create mode 100644 tests/integration_test.py create mode 100644 tests/unit_test.py delete mode 100644 unit_tests_scratch/.gitkeep diff --git a/LICENSE b/LICENSE index 9d05d34..0cf12e0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,6 @@ -Copyright (c) 2012, 2013, 2014, Fusionbox, Inc. All rights reserved. + BSD 2-Clause License + +Copyright (c) 2012-2015, Fusionbox, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in index fac5657..a5021c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include README.rst include LICENSE -include test_settings.py diff --git a/backupdb/utils/__init__.py b/backupdb/backends/__init__.py similarity index 100% rename from backupdb/utils/__init__.py rename to backupdb/backends/__init__.py diff --git a/backupdb/backends/base.py b/backupdb/backends/base.py new file mode 100644 index 0000000..700bbbe --- /dev/null +++ b/backupdb/backends/base.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import os +import abc + +import spm +import six + + +class BaseBackend(six.with_metaclass(abc.ABCMeta, object)): + def __init__(self, **kwargs): + self.db_config = kwargs.pop('db_config') + self.backup_file = kwargs.pop('backup_file') + self.extra_args = kwargs.pop('extra_args', []) + self.show_ouput = kwargs.pop('show_ouput', False) + + @abc.abstractmethod + def get_backup_command(self, db_name): + """ + Returns a spm command which dumps the database on stdout + """ + + @abc.abstractmethod + def get_restore_command(self, db_name, drop_tables): + """ + Returns a spm command which takes a backup file on stdin + """ + + def do_backup(self): + db_name = self.db_config['NAME'] # This has to raise KeyError if it does + # not exist. + + command = self.get_backup_command(db_name) + + proc = command.pipe('gzip') + proc.stdout = open(self.backup_file, 'w') + proc.wait() + + def do_restore(self, drop_tables): + db_name = self.db_config['NAME'] # This has to raise KeyError if it does + # not exist. + + if not os.path.isfile(self.backup_file): + raise RuntimeError("{} does not exist".format(fname)) + + command = self.get_restore_command(db_name, drop_tables=drop_tables) + + proc = spm.run('zcat', self.backup_file).pipe(command) + proc.wait() diff --git a/backupdb/backends/mysql.py b/backupdb/backends/mysql.py new file mode 100644 index 0000000..2727cf4 --- /dev/null +++ b/backupdb/backends/mysql.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import +import os + +import spm + +from backupdb.utils import apply_arg_values +from backupdb.backends.base import BaseBackend + + +class MySQLBackend(BaseBackend): + extension = 'mysql' + + def get_arguments(self): + return apply_arg_values( + ('--user={}', self.db_config.get('USER')), + ('--password={}', self.db_config.get('PASSWORD')), + ('--host={}', self.db_config.get('HOST')), + ('--port={}', self.db_config.get('PORT')) + ) + + def get_backup_command(self, db_name): + arguments = ['mysqldump'] + self.get_arguments() + self.extra_args + \ + [db_name] + return spm.run(*arguments) + + def get_restore_command(self, db_name, drop_tables): + # TODO: Drop all tables from the database if drop_tables is True + + arguments = ['mysql'] + self.get_arguments() + self.extra_args + \ + [db_name] + return spm.run(*arguments) diff --git a/backupdb/backends/postgresql.py b/backupdb/backends/postgresql.py new file mode 100644 index 0000000..8829d51 --- /dev/null +++ b/backupdb/backends/postgresql.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import +import os + +import spm + +from backupdb.utils import apply_arg_values +from backupdb.backends.base import BaseBackend + +# TODO: Use this command to drop tables from a database +PG_DROP_SQL = """SELECT format('DROP TABLE IF EXISTS %I CASCADE;', tablename) + FROM pg_tables WHERE schemaname = 'public';""" + + +class PostgreSQLBackend(BaseBackend): + extension = 'pgsql' + + def get_env(self): + try: + return {'PGPASSWORD': self.db_config['PASSWORD']} + except KeyError: + return {} + + def get_arguments(self): + return apply_arg_values( + ('--username={0}', self.db_config.get('USER')), + ('--host={0}', self.db_config.get('HOST')), + ('--port={0}', self.db_config.get('PORT')), + ) + + def get_backup_command(self): + arguments = ['pg_dump', '--clean'] + self.get_arguments() + \ + self.extra_args + [database] + env = self.get_env() + return spm.run(*arguments, env=env) + + def get_restore_command(self, drop_tables): + # TODO: Drop all tables from the database if drop_tables is True + database = self.db_config['NAME'] + + if not os.path.isfile(self.backup_file): + raise RuntimeError("{} does not exist".format(fname)) + + arguments = ['psql'] + self.get_arguments() + self.extra_args + \ + [database] + env = self.get_env() + + return spm.run(*arguments, env=env) diff --git a/backupdb/backends/sqlite.py b/backupdb/backends/sqlite.py new file mode 100644 index 0000000..afd5d99 --- /dev/null +++ b/backupdb/backends/sqlite.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import spm + +from backupdb.backends.base import BaseBackend + + +class SQLite3Backend(BaseBackend): + extension = 'sqlite' + + def get_backup_command(self, fname): + arguments = ['sqlite3'] + self.extra_args + [fname, '.dump'] + command = spm.run(*arguments) + + return command + + def get_restore_command(self, fname, drop_tables): + # TODO: Drop all tables from the database if drop_tables is True + return spm.run('sqlite3', fname) diff --git a/backupdb/tests/app/models.py b/backupdb/backends/sqlite3.py similarity index 100% rename from backupdb/tests/app/models.py rename to backupdb/backends/sqlite3.py diff --git a/backupdb/management/commands/backupdb.py b/backupdb/management/commands/backupdb.py index ffcf7d1..90a2b25 100644 --- a/backupdb/management/commands/backupdb.py +++ b/backupdb/management/commands/backupdb.py @@ -3,12 +3,13 @@ from subprocess import CalledProcessError import logging import os -import time +from datetime import datetime -from backupdb.utils.commands import BaseBackupDbCommand, do_postgresql_backup -from backupdb.utils.exceptions import BackupError -from backupdb.utils.log import section, SectionError, SectionWarning -from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG +from django.conf import settings +from django.utils import timezone + +from backupdb.utils import BaseBackupDbCommand +from backupdb.settings import get_backup_directory, BACKUP_CONFIG logger = logging.getLogger(__name__) @@ -51,48 +52,54 @@ class Command(BaseBackupDbCommand): def handle(self, *args, **options): super(Command, self).handle(*args, **options) - from django.conf import settings + current_time_string = timezone.now().isoformat() + + backup_suffix = options['backup_name'] + if backup_suffix is None: + backup_suffix = current_time_string - current_time = time.strftime('%F-%s') - backup_name = options['backup_name'] or current_time show_output = options['show_output'] + backup_directory = get_backup_directory() + # Ensure backup dir present - if not os.path.exists(BACKUP_DIR): - os.makedirs(BACKUP_DIR) + if not os.path.exists(backup_directory): + os.makedirs(backup_directory) # Loop through databases for db_name, db_config in settings.DATABASES.items(): - with section("Backing up '{0}'...".format(db_name)): - # Get backup config for this engine type - engine = db_config['ENGINE'] - backup_config = BACKUP_CONFIG.get(engine) - if not backup_config: - raise SectionWarning("Backup for '{0}' engine not implemented".format(engine)) - - # Get backup file name - backup_base_name = '{db_name}-{backup_name}.{backup_extension}.gz'.format( - db_name=db_name, - backup_name=backup_name, - backup_extension=backup_config['backup_extension'], - ) - backup_file = os.path.join(BACKUP_DIR, backup_base_name) - - # Find backup command and get kwargs - backup_func = backup_config['backup_func'] - backup_kwargs = { - 'backup_file': backup_file, - 'db_config': db_config, - 'show_output': show_output, - } - if backup_func is do_postgresql_backup: - backup_kwargs['pg_dump_options'] = options['pg_dump_options'] - - # Run backup command - try: - backup_func(**backup_kwargs) - logger.info("Backup of '{db_name}' saved in '{backup_file}'".format( + logger.info("Starting to backup {!r}".format(db_name)) + engine = db_config['ENGINE'] + try: + backup_cls = BACKUP_CONFIG[engine] + except KeyError: + logger.error( + "Backup for {!r} engine not implemented".format(engine)) + continue # TODO: Raise an error by default + # TODO: implement --ignore-errors + + fname = '{db}-{suffix}.{ext}.gz'.format( + db=db_name, + suffix=backup_suffix, + ext=backup_cls.extension, + ) + absolute_fname = os.path.join(backup_directory, fname) + + backup = backup_cls( + db_config=db_config, + backup_file=absolute_fname, + show_output=show_output, + ) + + + try: + backup.do_backup() + except RuntimeError as e: + logger.error(e.message) + else: + logger.info( + "Backup of {db_name!r} saved in {backup_file!r}".format( db_name=db_name, - backup_file=backup_file)) - except (BackupError, CalledProcessError) as e: - raise SectionError(e) + backup_file=fname + ) + ) diff --git a/backupdb/management/commands/restoredb.py b/backupdb/management/commands/restoredb.py index 87dc02d..11e4fee 100644 --- a/backupdb/management/commands/restoredb.py +++ b/backupdb/management/commands/restoredb.py @@ -8,11 +8,9 @@ from django.conf import settings from django.db import close_connection -from backupdb.utils.commands import BaseBackupDbCommand -from backupdb.utils.exceptions import RestoreError -from backupdb.utils.files import get_latest_timestamped_file -from backupdb.utils.log import section, SectionError, SectionWarning -from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG +from backupdb.utils import get_latest_timestamped_file +from backupdb.settings import get_backup_directory, BACKUP_CONFIG +from backupdb.utils import BaseBackupDbCommand logger = logging.getLogger(__name__) @@ -63,9 +61,12 @@ def handle(self, *args, **options): # Because of this psql can't drop django_content_types and just hangs close_connection() + backup_directory = get_backup_directory() + # Ensure backup dir present - if not os.path.exists(BACKUP_DIR): - raise CommandError("Backup dir '{0}' does not exist!".format(BACKUP_DIR)) + if not os.path.exists(backup_directory): + raise CommandError("Backup dir {!r} does not exist!".format( + backup_directory)) backup_name = options['backup_name'] drop_tables = options['drop_tables'] @@ -73,42 +74,44 @@ def handle(self, *args, **options): # Loop through databases for db_name, db_config in settings.DATABASES.items(): - with section("Restoring '{0}'...".format(db_name)): - # Get backup config for this engine type - engine = db_config['ENGINE'] - backup_config = BACKUP_CONFIG.get(engine) - if not backup_config: - raise SectionWarning("Restore for '{0}' engine not implemented".format(engine)) - - # Get backup file name - backup_extension = backup_config['backup_extension'] - if backup_name: - backup_file = '{dir}/{db_name}-{backup_name}.{ext}.gz'.format( - dir=BACKUP_DIR, - db_name=db_name, - backup_name=backup_name, - ext=backup_extension, + logger.info("Restoring {!r}".format(db_name)) + engine = db_config['ENGINE'] + try: + backup_cls = BACKUP_CONFIG[engine] + except KeyError: + logger.error( + "Restore for {!r} engine not implemented".format(engine)) + continue # TODO: Raise an error by default + # TODO: implement --ignore-errors + + + + if backup_name: + fname = '{db}-{suffix}.{ext}.gz'.format( + db=db_name, + suffix=backup_suffix, + ext=backup_cls.extension, + ) + else: + fname = get_latest_timestamped_file( + db_name, backup_cls.extension, backup_directory) + if fname is None: + logger.error( + "Couldn't find a default backup for '{!r}'".format( + db_name) ) - else: - try: - backup_file = get_latest_timestamped_file(backup_extension) - except RestoreError as e: - raise SectionError(e) - - # Find restore command and get kwargs - restore_func = backup_config['restore_func'] - restore_kwargs = { - 'backup_file': backup_file, - 'db_config': db_config, - 'drop_tables': drop_tables, - 'show_output': show_output, - } - - # Run restore command - try: - restore_func(**restore_kwargs) - logger.info("Restored '{db_name}' from '{backup_file}'".format( - db_name=db_name, - backup_file=backup_file)) - except (RestoreError, CalledProcessError) as e: - raise SectionError(e) + continue # TODO: Raise an error by default + # TODO: implement --ignore-errors + + absolute_fname = os.path.join(backup_directory, fname) + + backup = backup_cls( + db_config=db_config, + backup_file=absolute_fname, + show_output=show_output, + ) + + backup.do_restore(drop_tables=drop_tables) + logger.info("Restored {db_name!r} from {backup_file!r}".format( + db_name=db_name, + backup_file=absolute_fname)) diff --git a/backupdb/settings.py b/backupdb/settings.py new file mode 100644 index 0000000..547845f --- /dev/null +++ b/backupdb/settings.py @@ -0,0 +1,19 @@ +from django.conf import settings + +from backupdb.backends.mysql import MySQLBackend +from backupdb.backends.sqlite import SQLite3Backend +from backupdb.backends.postgresql import PostgreSQLBackend + + +DEFAULT_BACKUP_DIR = 'backups' + +BACKUP_CONFIG = { + 'django.db.backends.mysql': MySQLBackend, + 'django.db.backends.postgresql_psycopg2': PostgreSQLBackend, + 'django.contrib.gis.db.backends.postgis': PostgreSQLBackend, + 'django.db.backends.sqlite3': SQLite3Backend, +} + + +def get_backup_directory(): + return getattr(settings, 'BACKUPDB_DIRECTORY', DEFAULT_BACKUP_DIR) diff --git a/backupdb/tests/__init__.py b/backupdb/tests/__init__.py deleted file mode 100644 index 7c36d70..0000000 --- a/backupdb/tests/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -from . import commands -from . import files -from . import log -from . import processes - - -loader = unittest.TestLoader() - -commands_tests = loader.loadTestsFromModule(commands) -files_tests = loader.loadTestsFromModule(files) -log_tests = loader.loadTestsFromModule(log) -processes_tests = loader.loadTestsFromModule(processes) - -all_tests = unittest.TestSuite([ - commands_tests, - files_tests, - log_tests, - processes_tests, -]) diff --git a/backupdb/tests/app/urls.py b/backupdb/tests/app/urls.py deleted file mode 100644 index 41a2be0..0000000 --- a/backupdb/tests/app/urls.py +++ /dev/null @@ -1,2 +0,0 @@ - -urlpatterns = () diff --git a/backupdb/tests/commands.py b/backupdb/tests/commands.py deleted file mode 100644 index 7dd713b..0000000 --- a/backupdb/tests/commands.py +++ /dev/null @@ -1,367 +0,0 @@ -from mock import call, patch -import unittest - -from backupdb.utils.commands import ( - PG_DROP_SQL, - get_mysql_args, - get_postgresql_args, - get_postgresql_env, - do_mysql_backup, - do_postgresql_backup, - do_sqlite_backup, - do_mysql_restore, - do_postgresql_restore, - do_sqlite_restore, -) -from backupdb.utils.exceptions import RestoreError - - -DB_CONFIG = { - 'NAME': 'test_db', - 'USER': 'test_user', - 'PASSWORD': 'test_password', - 'HOST': 'test_host', - 'PORT': 12345, -} - - -def make_db_config(*keys): - new_dict = {} - for k in keys: - new_dict[k] = DB_CONFIG[k] - return new_dict - - -class MockOsPathExists(object): - """ - Used as a mock object for os.path.exists. - """ - def __init__(self, *args, **kwargs): - pass - - def __call__(self, *args, **kwargs): - return True - - -class PatchPipeCommandsTestCase(unittest.TestCase): - """ - Used for testing of pipe_commands and pipe_commands_to_file. - """ - def setUp(self): - self.pipe_commands_patcher = patch('backupdb.utils.commands.pipe_commands') - self.pipe_commands_to_file_patcher = patch('backupdb.utils.commands.pipe_commands_to_file') - self.os_patcher = patch('os.path.exists', new_callable=MockOsPathExists) - - self.mock_pipe_commands = self.pipe_commands_patcher.start() - self.mock_pipe_commands_to_file = self.pipe_commands_to_file_patcher.start() - self.mock_os = self.os_patcher.start() - - def tearDown(self): - self.pipe_commands_patcher.stop() - self.pipe_commands_to_file_patcher.stop() - self.os_patcher.stop() - - def assertPipeCommandsCallsEqual(self, *args): - self.assertEqual(self.mock_pipe_commands.call_args_list, list(args)) - - def assertPipeCommandsToFileCallsEqual(self, *args): - self.assertEqual(self.mock_pipe_commands_to_file.call_args_list, list(args)) - - -class RequireBackupExistsTestCase(unittest.TestCase): - def test_it_raises_an_exception_when_the_path_in_backup_file_arg_doesnt_exist(self): - self.assertRaises(RestoreError, do_mysql_restore, backup_file='i_dont_exist', db_config={}) - self.assertRaises(RestoreError, do_postgresql_restore, backup_file='i_dont_exist', db_config={}) - self.assertRaises(RestoreError, do_sqlite_restore, backup_file='i_dont_exist', db_config={}) - - -class GetMysqlArgsTestCase(unittest.TestCase): - def test_it_builds_the_correct_args(self): - self.assertEqual( - get_mysql_args(make_db_config('NAME', 'USER')), - [ - '--user=test_user', - 'test_db', - ], - ) - self.assertEqual( - get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD')), - [ - '--user=test_user', - '--password=test_password', - 'test_db', - ], - ) - self.assertEqual( - get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD', 'HOST')), - [ - '--user=test_user', - '--password=test_password', - '--host=test_host', - 'test_db', - ], - ) - self.assertEqual( - get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD', 'HOST', 'PORT')), - [ - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ) - - -class GetPostgresqlArgsTestCase(unittest.TestCase): - def test_it_builds_the_correct_args(self): - self.assertEqual( - get_postgresql_args(make_db_config('NAME', 'USER')), - [ - '--username=test_user', - 'test_db', - ], - ) - self.assertEqual( - get_postgresql_args(make_db_config('NAME', 'USER', 'HOST')), - [ - '--username=test_user', - '--host=test_host', - 'test_db', - ], - ) - self.assertEqual( - get_postgresql_args(make_db_config('NAME', 'USER', 'HOST', 'PORT')), - [ - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ) - - def test_it_correctly_includes_extra_args(self): - self.assertEqual( - get_postgresql_args( - make_db_config('NAME', 'USER'), - extra_args='--no-owner --no-privileges', - ), - [ - '--username=test_user', - '--no-owner', - '--no-privileges', - 'test_db', - ], - ) - - -class GetPostgresqlEnvTestCase(unittest.TestCase): - def test_it_builds_the_correct_env_dict(self): - self.assertTrue( - get_postgresql_env(make_db_config('USER', 'NAME')) is None, - ) - self.assertEqual( - get_postgresql_env(make_db_config('USER', 'NAME', 'PASSWORD')), - {'PGPASSWORD': 'test_password'}, - ) - - -class DoMysqlBackupTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_mysql_backup('test.mysql.gz', DB_CONFIG) - - self.assertPipeCommandsToFileCallsEqual(call( - [ - [ - 'mysqldump', - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ['gzip'], - ], - path='test.mysql.gz', - show_stderr=False, - )) - - -class DoPostgresqlBackupTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_postgresql_backup('test.pgsql.gz', DB_CONFIG) - - self.assertPipeCommandsToFileCallsEqual(call( - [ - [ - 'pg_dump', - '--clean', - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ['gzip'], - ], - path='test.pgsql.gz', - extra_env={'PGPASSWORD': 'test_password'}, - show_stderr=False, - )) - - -class DoSqliteBackupTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_sqlite_backup('test.sqlite.gz', DB_CONFIG) - - self.assertPipeCommandsToFileCallsEqual(call( - [['cat', 'test_db'], ['gzip']], - path='test.sqlite.gz', - show_stderr=False, - )) - - -class DoMysqlRestoreTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_mysql_restore(backup_file='test.mysql.gz', db_config=DB_CONFIG) - - self.assertPipeCommandsCallsEqual(call( - [ - ['cat', 'test.mysql.gz'], - ['gunzip'], - [ - 'mysql', - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - show_stderr=False, - show_last_stdout=False, - )) - - def test_it_drops_tables_before_restoring_if_specified(self): - do_mysql_restore(backup_file='test.mysql.gz', db_config=DB_CONFIG, drop_tables=True) - - self.assertPipeCommandsCallsEqual( - call( - [ - [ - 'mysqldump', - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - '--no-data', - ], - ['grep', '^DROP'], - [ - 'mysql', - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - show_stderr=False, - show_last_stdout=False, - ), - call( - [ - ['cat', 'test.mysql.gz'], - ['gunzip'], - [ - 'mysql', - '--user=test_user', - '--password=test_password', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - show_stderr=False, - show_last_stdout=False, - ), - ) - - -class DoPostgresqlRestoreTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_postgresql_restore(backup_file='test.pgsql.gz', db_config=DB_CONFIG) - - self.assertPipeCommandsCallsEqual(call( - [ - ['cat', 'test.pgsql.gz'], - ['gunzip'], - [ - 'psql', - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - extra_env={'PGPASSWORD': 'test_password'}, - show_stderr=False, - show_last_stdout=False, - )) - - def test_it_drops_tables_before_restoring_if_specified(self): - do_postgresql_restore(backup_file='test.pgsql.gz', db_config=DB_CONFIG, drop_tables=True) - - self.assertPipeCommandsCallsEqual( - call( - [ - [ - 'psql', - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - '-t', - '-c', - PG_DROP_SQL, - ], - [ - 'psql', - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - extra_env={'PGPASSWORD': 'test_password'}, - show_stderr=False, - show_last_stdout=False, - ), - call( - [ - ['cat', 'test.pgsql.gz'], - ['gunzip'], - [ - 'psql', - '--username=test_user', - '--host=test_host', - '--port=12345', - 'test_db', - ], - ], - extra_env={'PGPASSWORD': 'test_password'}, - show_stderr=False, - show_last_stdout=False, - ), - ) - - -class DoSqliteRestoreTestCase(PatchPipeCommandsTestCase): - def test_it_makes_correct_calls_to_processes_api(self): - do_sqlite_restore(backup_file='test.sqlite.gz', db_config=DB_CONFIG) - - self.assertPipeCommandsToFileCallsEqual(call( - [['cat', 'test.sqlite.gz'], ['gunzip']], - path='test_db', - show_stderr=False, - )) diff --git a/backupdb/tests/files.py b/backupdb/tests/files.py deleted file mode 100644 index 7eecf4f..0000000 --- a/backupdb/tests/files.py +++ /dev/null @@ -1,37 +0,0 @@ -from backupdb.utils.exceptions import RestoreError -from backupdb.utils.files import get_latest_timestamped_file - -from backupdb.tests.utils import FileSystemScratchTestCase - - -class GetLatestTimestampedFileTestCase(FileSystemScratchTestCase): - def create_files(self, *args): - for file in args: - open(self.get_path(file), 'a').close() - - def test_it_returns_the_latest_timestamped_file_with_ext(self): - self.create_files( - 'default-2013-05-02-1367553089.sqlite.gz', - 'default-2013-06-06-1370570260.sqlite.gz', - 'default-2013-06-06-1370580510.sqlite.gz', - - 'default-2013-05-02-1367553089.mysql.gz', - 'default-2013-06-06-1370570260.mysql.gz', - 'default-2013-06-06-1370580510.mysql.gz', - - 'default-2013-05-02-1367553089.pgsql.gz', - 'default-2013-06-06-1370570260.pgsql.gz', - 'default-2013-06-06-1370580510.pgsql.gz', - ) - - sqlite_file = get_latest_timestamped_file('sqlite', dir=self.SCRATCH_DIR) - mysql_file = get_latest_timestamped_file('mysql', dir=self.SCRATCH_DIR) - pgsql_file = get_latest_timestamped_file('pgsql', dir=self.SCRATCH_DIR) - - self.assertEqual(sqlite_file, self.get_path('default-2013-06-06-1370580510.sqlite.gz')) - self.assertEqual(mysql_file, self.get_path('default-2013-06-06-1370580510.mysql.gz')) - self.assertEqual(pgsql_file, self.get_path('default-2013-06-06-1370580510.pgsql.gz')) - - def test_it_raises_an_exception_when_no_files_found(self): - self.assertRaises(RestoreError, get_latest_timestamped_file, '', dir=self.SCRATCH_DIR) - self.assertRaises(RestoreError, get_latest_timestamped_file, 'mysql', dir=self.SCRATCH_DIR) diff --git a/backupdb/tests/log.py b/backupdb/tests/log.py deleted file mode 100644 index 9a3969c..0000000 --- a/backupdb/tests/log.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest - -from backupdb.utils.log import bar - - -class BarTestCase(unittest.TestCase): - def test_it_should_return_a_string_with_text_centered_in_a_bar(self): - self.assertEqual( - bar('test', width=70), - '================================ test ================================', - ) - - def test_it_should_return_a_bar_string_with_the_specified_width(self): - test_bar1 = r'=========================== test ===========================' - test_bar2 = r'//========================= test =========================\\' - test_bar3 = r'\\========================= test =========================//' - - self.assertEqual(len(test_bar1), 60) - self.assertEqual(len(test_bar2), 60) - self.assertEqual(len(test_bar3), 60) - - self.assertEqual(bar('test', width=60), test_bar1) - self.assertEqual(bar('test', width=60, position='top'), test_bar2) - self.assertEqual(bar('test', width=60, position='bottom'), test_bar3) - - def test_it_should_work_even_if_the_given_width_is_less_than_the_message_length(self): - self.assertEqual(bar('test', width=0), '== test ==') - self.assertEqual(bar('test', width=0, position='top'), r'// test \\') - self.assertEqual(bar('test', width=0, position='bottom'), r'\\ test //') - - def test_it_should_render_a_top_bottom_or_plain_bar_depending_on_the_position_argument(self): - test_bar1 = r'================================ test ================================' - test_bar2 = r'//============================== test ==============================\\' - test_bar3 = r'\\============================== test ==============================//' - - self.assertEqual(bar('test', width=70), test_bar1) - self.assertEqual(bar('test', width=70, position='top'), test_bar2) - self.assertEqual(bar('test', width=70, position='bottom'), test_bar3) - - def test_it_should_allow_the_message_to_be_blank(self): - test_bar1 = r'======================================================================' - test_bar2 = r'//==================================================================\\' - test_bar3 = r'\\==================================================================//' - - self.assertEqual(bar(width=70), test_bar1) - self.assertEqual(bar(width=70, position='top'), test_bar2) - self.assertEqual(bar(width=70, position='bottom'), test_bar3) diff --git a/backupdb/tests/processes.py b/backupdb/tests/processes.py deleted file mode 100644 index 3b705f6..0000000 --- a/backupdb/tests/processes.py +++ /dev/null @@ -1,170 +0,0 @@ -from subprocess import CalledProcessError -import os -import unittest - -try: - from collections import OrderedDict -except ImportError: - # This should only happen in Python 2.6 - # SortedDict is deprecated in Django 1.7 and will be removed in Django 1.9 - from django.utils.datastructures import SortedDict as OrderedDict - -from backupdb.utils.processes import ( - extend_env, - get_env_str, - pipe_commands, - pipe_commands_to_file, -) - -from .utils import FileSystemScratchTestCase - - -class ExtendEnvTestCase(unittest.TestCase): - def test_extend_env_creates_a_copy_of_the_current_env(self): - env = extend_env({'BACKUPDB_TEST_ENV_SETTING': 1234}) - self.assertFalse(env is os.environ) - - def test_extend_env_adds_keys_to_a_copy_of_the_current_env(self): - env = extend_env({ - 'BACKUPDB_TEST_ENV_SETTING_1': 1234, - 'BACKUPDB_TEST_ENV_SETTING_2': 1234, - }) - - orig_keys = set(os.environ.keys()) - curr_keys = set(env.keys()) - diff_keys = curr_keys - orig_keys - - self.assertEqual(diff_keys, set([ - 'BACKUPDB_TEST_ENV_SETTING_1', - 'BACKUPDB_TEST_ENV_SETTING_2', - ])) - - -class GetEnvStrTestCase(unittest.TestCase): - def test_get_env_str_works_for_empty_dicts(self): - self.assertEqual(get_env_str({}), '') - - def test_get_env_str_works_for_non_empty_dicts(self): - self.assertEqual( - get_env_str({'VAR_1': 1234}), - "VAR_1='1234'", - ) - self.assertEqual( - get_env_str(OrderedDict([ - ('VAR_1', 1234), - ('VAR_2', 'arst'), - ])), - "VAR_1='1234' VAR_2='arst'", - ) - self.assertEqual( - get_env_str(OrderedDict([ - ('VAR_1', 1234), - ('VAR_2', 'arst'), - ('VAR_3', 'zxcv'), - ])), - "VAR_1='1234' VAR_2='arst' VAR_3='zxcv'", - ) - - -class PipeCommandsTestCase(FileSystemScratchTestCase): - def test_it_pipes_a_list_of_commands_into_each_other(self): - pipe_commands([ - ['echo', r""" -import sys -for i in range(4): - sys.stdout.write('spam\n') -"""], - ['python'], - ['tee', self.get_path('pipe_commands.out')], - ]) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasContent( - 'pipe_commands.out', - 'spam\nspam\nspam\nspam\n', - ) - - def test_it_works_when_large_amounts_of_data_are_being_piped(self): - pipe_commands([ - ['echo', r""" -import sys -for i in range(400000): - sys.stdout.write('spam\n') -"""], - ['python'], - ['tee', self.get_path('pipe_commands.out')], - ]) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasLength('pipe_commands.out', 2000000) - self.assertInFile('pipe_commands.out', 'spam\nspam\nspam\n') - - def test_it_allows_you_to_specify_extra_environment_variables(self): - pipe_commands([ - ['echo', """ -import os -import sys -sys.stdout.write(os.environ['TEST_VAR']) -"""], - ['python'], - ['tee', self.get_path('pipe_commands.out')], - ], extra_env={'TEST_VAR': 'spam'}) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasContent('pipe_commands.out', 'spam') - - def test_it_correctly_raises_a_called_process_error_when_necessary(self): - self.assertRaises(CalledProcessError, pipe_commands, [['false'], ['true']]) - - -class PipeCommandsToFileTestCase(FileSystemScratchTestCase): - def test_it_pipes_a_list_of_commands_into_each_other_and_then_into_a_file(self): - pipe_commands_to_file([ - ['echo', r""" -import sys -for i in range(4): - sys.stdout.write('spam\n') -"""], - ['python'], - ], self.get_path('pipe_commands.out')) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasContent( - 'pipe_commands.out', - 'spam\nspam\nspam\nspam\n', - ) - - def test_it_works_when_large_amounts_of_data_are_being_piped(self): - pipe_commands_to_file([ - ['echo', r""" -import sys -for i in range(400000): - sys.stdout.write('spam\n') -"""], - ['python'], - ], self.get_path('pipe_commands.out')) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasLength('pipe_commands.out', 2000000) - self.assertInFile('pipe_commands.out', 'spam\nspam\nspam\n') - - def test_it_allows_you_to_specify_extra_environment_variables(self): - pipe_commands_to_file([ - ['echo', """ -import os -import sys -sys.stdout.write(os.environ['TEST_VAR']) -"""], - ['python'], - ], self.get_path('pipe_commands.out'), extra_env={'TEST_VAR': 'spam'}) - - self.assertFileExists('pipe_commands.out') - self.assertFileHasContent('pipe_commands.out', 'spam') - - def test_it_correctly_raises_a_called_process_error_when_necessary(self): - self.assertRaises( - CalledProcessError, - pipe_commands_to_file, - [['false'], ['true']], - self.get_path('pipe_commands.out'), - ) diff --git a/backupdb/tests/utils.py b/backupdb/tests/utils.py deleted file mode 100644 index 73619d6..0000000 --- a/backupdb/tests/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import unittest - - -class FileSystemScratchTestCase(unittest.TestCase): - SCRATCH_DIR = 'unit_tests_scratch' - - @classmethod - def get_path(cls, file): - return os.path.join(cls.SCRATCH_DIR, file) - - @classmethod - def clear_scratch_dir(cls): - """ - Deletes all scratch files in the tests scratch directory. - """ - for file in os.listdir(cls.SCRATCH_DIR): - if file != '.gitkeep': - os.remove(cls.get_path(file)) - - def setUp(self): - self.clear_scratch_dir() - - def tearDown(self): - self.clear_scratch_dir() - - def assertFileExists(self, file): - self.assertTrue(os.path.exists(self.get_path(file))) - - def assertFileHasLength(self, file, length): - with open(self.get_path(file)) as f: - content = f.read() - self.assertEqual(len(content), length) - - def assertFileHasContent(self, file, expected_content): - with open(self.get_path(file)) as f: - actual_content = f.read() - self.assertEqual(actual_content, expected_content) - - def assertInFile(self, file, expected_content): - with open(self.get_path(file)) as f: - actual_content = f.read() - self.assertTrue(expected_content in actual_content) diff --git a/backupdb/utils.py b/backupdb/utils.py new file mode 100644 index 0000000..ec54225 --- /dev/null +++ b/backupdb/utils.py @@ -0,0 +1,65 @@ +import os +import glob +import re +import logging + +from django.core.management.base import BaseCommand + +TIMESTAMP_PATTERNS = ( + r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}', # New ISO format pattern + r'\d{4}-\d{2}-\d{2}-\d{10}', # Old style pattern +) + + +def get_latest_timestamped_file(name, ext, directory): + """ + Gets the latest timestamped backup file name with the given database type + extension. + """ + for pat in TIMESTAMP_PATTERNS: + matcher = re.compile(r'^{}-{}\.{}'.format( + re.escape(name), pat, re.escape(ext))) + + files = [f for f in os.listdir(directory) if matcher.match(f)] + files.sort(reverse=True) + + try: + return files[0] + except IndexError: + continue + + return None + + +def apply_arg_values(*args): + """ + Apply argument to values:: + + >>> apply_arg_values(('--name={}', 'name'), + ... ('--password={}', 'password'), + ... ('--level={}', ''), + ... ('--last={}', None)) + ['--name=name', '--password=password'] + + """ + return [a.format(v) for a, v in args if v] + + +# TODO: Get rid of BaseBackupDbCommand +class BaseBackupDbCommand(BaseCommand): + can_import_settings = True + + LOG_LEVELS = { + 0: logging.ERROR, + 1: logging.INFO, + 2: logging.DEBUG, + 3: logging.DEBUG, + } + LOG_FORMAT = '%(asctime)s - %(levelname)-8s: %(message)s' + + def _setup_logging(self, level): + level = int(level) + logging.basicConfig(format=self.LOG_FORMAT, level=self.LOG_LEVELS[level]) + + def handle(self, *args, **options): + self._setup_logging(options['verbosity']) diff --git a/backupdb/utils/commands.py b/backupdb/utils/commands.py deleted file mode 100644 index d35249d..0000000 --- a/backupdb/utils/commands.py +++ /dev/null @@ -1,163 +0,0 @@ -import logging -import os -import shlex - -from django.core.management.base import BaseCommand - -from .exceptions import RestoreError -from .processes import pipe_commands, pipe_commands_to_file - - -PG_DROP_SQL = """SELECT 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE;' FROM pg_tables WHERE schemaname = 'public';""" - - -class BaseBackupDbCommand(BaseCommand): - can_import_settings = True - - LOG_LEVELS = { - 0: logging.ERROR, - 1: logging.INFO, - 2: logging.DEBUG, - 3: logging.DEBUG, - } - LOG_FORMAT = '%(asctime)s - %(levelname)-8s: %(message)s' - - def _setup_logging(self, level): - level = int(level) - logging.basicConfig(format=self.LOG_FORMAT, level=self.LOG_LEVELS[level]) - - def handle(self, *args, **options): - self._setup_logging(options['verbosity']) - - -def apply_arg_values(arg_values): - """ - Apply argument to values. - - l = [('--name={0}', 'name'), - ('--password={0}', 'password'), - ('--level={0}', ''), - ('--last={0}', None)] - assert apply_arg_values(l) == ['--name=name', '--password=password'] - """ - return [a.format(v) for a, v in arg_values if v] - - -def require_backup_exists(func): - """ - Requires that the file referred to by `backup_file` exists in the file - system before running the decorated function. - """ - def new_func(*args, **kwargs): - backup_file = kwargs['backup_file'] - if not os.path.exists(backup_file): - raise RestoreError("Could not find file '{0}'".format(backup_file)) - return func(*args, **kwargs) - return new_func - - -def get_mysql_args(db_config): - """ - Returns an array of argument values that will be passed to a `mysql` or - `mysqldump` process when it is started based on the given database - configuration. - """ - db = db_config['NAME'] - - mapping = [('--user={0}', db_config.get('USER')), - ('--password={0}', db_config.get('PASSWORD')), - ('--host={0}', db_config.get('HOST')), - ('--port={0}', db_config.get('PORT'))] - args = apply_arg_values(mapping) - args.append(db) - - return args - - -def get_postgresql_args(db_config, extra_args=None): - """ - Returns an array of argument values that will be passed to a `psql` or - `pg_dump` process when it is started based on the given database - configuration. - """ - db = db_config['NAME'] - - mapping = [('--username={0}', db_config.get('USER')), - ('--host={0}', db_config.get('HOST')), - ('--port={0}', db_config.get('PORT'))] - args = apply_arg_values(mapping) - - if extra_args is not None: - args.extend(shlex.split(extra_args)) - args.append(db) - - return args - - -def get_postgresql_env(db_config): - """ - Returns a dict containing extra environment variable values that will be - added to the environment of the `psql` or `pg_dump` process when it is - started based on the given database configuration. - """ - password = db_config.get('PASSWORD') - return {'PGPASSWORD': password} if password else None - - -def do_mysql_backup(backup_file, db_config, show_output=False): - args = get_mysql_args(db_config) - - cmd = ['mysqldump'] + args - pipe_commands_to_file([cmd, ['gzip']], path=backup_file, show_stderr=show_output) - - -def do_postgresql_backup(backup_file, db_config, pg_dump_options=None, show_output=False): - env = get_postgresql_env(db_config) - args = get_postgresql_args(db_config, pg_dump_options) - - cmd = ['pg_dump', '--clean'] + args - pipe_commands_to_file([cmd, ['gzip']], path=backup_file, extra_env=env, show_stderr=show_output) - - -def do_sqlite_backup(backup_file, db_config, show_output=False): - db_file = db_config['NAME'] - - cmd = ['cat', db_file] - pipe_commands_to_file([cmd, ['gzip']], path=backup_file, show_stderr=show_output) - - -@require_backup_exists -def do_mysql_restore(backup_file, db_config, drop_tables=False, show_output=False): - args = get_mysql_args(db_config) - mysql_cmd = ['mysql'] + args - - kwargs = {'show_stderr': show_output, 'show_last_stdout': show_output} - - if drop_tables: - dump_cmd = ['mysqldump'] + args + ['--no-data'] - pipe_commands([dump_cmd, ['grep', '^DROP'], mysql_cmd], **kwargs) - - pipe_commands([['cat', backup_file], ['gunzip'], mysql_cmd], **kwargs) - - -@require_backup_exists -def do_postgresql_restore(backup_file, db_config, drop_tables=False, show_output=False): - env = get_postgresql_env(db_config) - args = get_postgresql_args(db_config) - psql_cmd = ['psql'] + args - - kwargs = {'extra_env': env, 'show_stderr': show_output, 'show_last_stdout': show_output} - - if drop_tables: - gen_drop_sql_cmd = psql_cmd + ['-t', '-c', PG_DROP_SQL] - pipe_commands([gen_drop_sql_cmd, psql_cmd], **kwargs) - - pipe_commands([['cat', backup_file], ['gunzip'], psql_cmd], **kwargs) - - -@require_backup_exists -def do_sqlite_restore(backup_file, db_config, drop_tables=False, show_output=False): - db_file = db_config['NAME'] - - cmd = ['cat', backup_file] - pipe_commands_to_file([cmd, ['gunzip']], path=db_file, show_stderr=show_output) diff --git a/backupdb/utils/exceptions.py b/backupdb/utils/exceptions.py deleted file mode 100644 index 98d0627..0000000 --- a/backupdb/utils/exceptions.py +++ /dev/null @@ -1,6 +0,0 @@ -class BackupError(Exception): - pass - - -class RestoreError(Exception): - pass diff --git a/backupdb/utils/files.py b/backupdb/utils/files.py deleted file mode 100644 index d409547..0000000 --- a/backupdb/utils/files.py +++ /dev/null @@ -1,25 +0,0 @@ -import glob - -from .exceptions import RestoreError -from .settings import BACKUP_DIR, BACKUP_TIMESTAMP_PATTERN - - -def get_latest_timestamped_file(ext, dir=BACKUP_DIR, pattern=BACKUP_TIMESTAMP_PATTERN): - """ - Gets the latest timestamped backup file name with the given database type - extension. - """ - pattern = '{dir}/{pattern}.{ext}.gz'.format( - dir=dir, - pattern=pattern, - ext=ext, - ) - - l = glob.glob(pattern) - l.sort() - l.reverse() - - if not l: - raise RestoreError("No backups found matching '{0}' pattern".format(pattern)) - - return l[0] diff --git a/backupdb/utils/log.py b/backupdb/utils/log.py deleted file mode 100644 index 5446c39..0000000 --- a/backupdb/utils/log.py +++ /dev/null @@ -1,67 +0,0 @@ -import contextlib -import logging - -logger = logging.getLogger(__name__) - - -def bar(msg='', width=40, position=None): - r""" - Returns a string with text centered in a bar caption. - - Examples: - >>> bar('test', width=10) - '== test ==' - >>> bar(width=10) - '==========' - >>> bar('Richard Dean Anderson is...', position='top', width=50) - '//========= Richard Dean Anderson is... ========\\\\' - >>> bar('...MacGyver', position='bottom', width=50) - '\\\\================= ...MacGyver ================//' - """ - if position == 'top': - start_bar = '//' - end_bar = r'\\' - elif position == 'bottom': - start_bar = r'\\' - end_bar = '//' - else: - start_bar = end_bar = '==' - - if msg: - msg = ' ' + msg + ' ' - - width -= 4 - - return start_bar + msg.center(width, '=') + end_bar - - -class SectionError(Exception): - pass - - -class SectionWarning(Exception): - pass - - -@contextlib.contextmanager -def section(msg): - """ - Context manager that prints a top bar to stderr upon entering and a bottom - bar upon exiting. The caption of the top bar is specified by `msg`. The - caption of the bottom bar is '...done!' if the context manager exits - successfully. If a SectionError or SectionWarning is raised inside of the - context manager, SectionError.message or SectionWarning.message is passed - to logging.error or logging.warning respectively and the bottom bar caption - becomes '...skipped.'. - """ - logger.info(bar(msg, position='top')) - try: - yield - except SectionError as e: - logger.error(e.message) - logger.info(bar('...skipped.', position='bottom')) - except SectionWarning as e: - logger.warning(e.message) - logger.info(bar('...skipped.', position='bottom')) - else: - logger.info(bar('...done!', position='bottom')) diff --git a/backupdb/utils/processes.py b/backupdb/utils/processes.py deleted file mode 100644 index 8142703..0000000 --- a/backupdb/utils/processes.py +++ /dev/null @@ -1,96 +0,0 @@ -from subprocess import Popen, PIPE, CalledProcessError -import logging -import os -import shutil - -logger = logging.getLogger(__name__) - - -def extend_env(extra_env): - """ - Copies and extends the current environment with the values present in - `extra_env`. - """ - env = os.environ.copy() - env.update(extra_env) - return env - - -def get_env_str(env): - """ - Gets a string representation of a dict as though it contained environment - variable values. - """ - return ' '.join("{0}='{1}'".format(k, v) for k, v in env.items()) - - -def pipe_commands(cmds, extra_env=None, show_stderr=False, show_last_stdout=False): - """ - Executes the list of commands piping each one into the next. - """ - env = extend_env(extra_env) if extra_env else None - env_str = (get_env_str(extra_env) + ' ') if extra_env else '' - cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds] - - logger.info('Running `{0}`'.format(' | '.join(cmd_strs))) - - with open('/dev/null', 'w') as NULL: - # Start processes - processes = [] - last_i = len(cmds) - 1 - for i, (cmd_str, cmd) in enumerate(zip(cmd_strs, cmds)): - if i == last_i: - p_stdout = None if show_last_stdout else NULL - else: - p_stdout = PIPE - p_stdin = processes[-1][1].stdout if processes else None - p_stderr = None if show_stderr else NULL - - p = Popen(cmd, env=env, stdout=p_stdout, stdin=p_stdin, stderr=p_stderr) - processes.append((cmd_str, p)) - - # Close processes - error = False - for cmd_str, p in processes: - if p.stdout: - p.stdout.close() - if p.wait() != 0: - error = True - if error: - raise CalledProcessError(cmd=cmd_str, returncode=p.returncode) - - -def pipe_commands_to_file(cmds, path, extra_env=None, show_stderr=False): - """ - Executes the list of commands piping each one into the next and writing - stdout of the last process into a file at the given path. - """ - env = extend_env(extra_env) if extra_env else None - env_str = (get_env_str(extra_env) + ' ') if extra_env else '' - cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds] - - logger.info('Saving output of `{0}`'.format(' | '.join(cmd_strs))) - - with open('/dev/null', 'w') as NULL: - # Start processes - processes = [] - for cmd_str, cmd in zip(cmd_strs, cmds): - p_stdin = processes[-1][1].stdout if processes else None - p_stderr = None if show_stderr else NULL - - p = Popen(cmd, env=env, stdout=PIPE, stdin=p_stdin, stderr=p_stderr) - processes.append((cmd_str, p)) - - p_last = processes[-1][1] - - with open(path, 'wb') as f: - shutil.copyfileobj(p_last.stdout, f) - - # Close processes - error = False - for cmd_str, p in processes: - p.stdout.close() - if p.wait() != 0: - error = True - if error: - raise CalledProcessError(cmd=cmd_str, returncode=p.returncode) diff --git a/backupdb/utils/settings.py b/backupdb/utils/settings.py deleted file mode 100644 index 19fc5c6..0000000 --- a/backupdb/utils/settings.py +++ /dev/null @@ -1,36 +0,0 @@ -from .commands import ( - do_mysql_backup, - do_mysql_restore, - do_postgresql_backup, - do_postgresql_restore, - do_sqlite_backup, - do_sqlite_restore, -) -from django.conf import settings - - -DEFAULT_BACKUP_DIR = 'backups' -BACKUP_DIR = getattr(settings, 'BACKUPDB_DIRECTORY', DEFAULT_BACKUP_DIR) -BACKUP_TIMESTAMP_PATTERN = '*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' -BACKUP_CONFIG = { - 'django.db.backends.mysql': { - 'backup_extension': 'mysql', - 'backup_func': do_mysql_backup, - 'restore_func': do_mysql_restore, - }, - 'django.db.backends.postgresql_psycopg2': { - 'backup_extension': 'pgsql', - 'backup_func': do_postgresql_backup, - 'restore_func': do_postgresql_restore, - }, - 'django.contrib.gis.db.backends.postgis': { - 'backup_extension': 'pgsql', - 'backup_func': do_postgresql_backup, - 'restore_func': do_postgresql_restore, - }, - 'django.db.backends.sqlite3': { - 'backup_extension': 'sqlite', - 'backup_func': do_sqlite_backup, - 'restore_func': do_sqlite_restore, - }, -} diff --git a/setup.py b/setup.py index a402881..9cb0640 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ current_path = os.path.dirname(__file__) sys.path.append(current_path) -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_settings') def get_version(): @@ -29,6 +28,8 @@ def get_readme(): with open(os.path.join(current_path, 'README.rst'), 'r') as f: return f.read() +# TODO: Integrate py.test in python setup.py test + setup( name='django-backupdb', version=get_version(), @@ -38,22 +39,24 @@ def get_readme(): author_email='programmers@fusionbox.com', keywords='django database backup', url='https://github.com/fusionbox/django-backupdb', - packages=['backupdb', 'backupdb.utils', 'backupdb.management', - 'backupdb.management.commands', 'backupdb.tests', - 'backupdb.tests.app'], + packages=['backupdb', 'backupdb.backends', 'backupdb.management', + 'backupdb.management.commands'], platforms='any', - license='Fusionbox', - test_suite='backupdb.tests.all_tests', + license='BSD', tests_require=[ - 'mock>=1.0.1', 'dj_database_url==0.3.0', ], install_requires=[ 'Django>=1.4', + 'spm', # This avoid implementing subcommands in backupdb ], classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 4 - Beta', 'Environment :: Web Environment', 'Framework :: Django', + 'License :: OSI Approved :: BSD License' + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', ], ) diff --git a/test_settings.py b/test_settings.py deleted file mode 100644 index 5492d7d..0000000 --- a/test_settings.py +++ /dev/null @@ -1,23 +0,0 @@ -import atexit -import dj_database_url -import os -import shutil -import tempfile - -DEBUG = True - -DATABASE = { - 'default': dj_database_url.config(default='sqlite://:memory:'), -} - -INSTALLED_APPS = ( - 'backupdb', - 'backupdb.tests.app', -) - -SECRET_KEY = 'this is a secret!' - -BACKUPDB_DIRECTORY = tempfile.mkdtemp() - -# Cleanup after itself -atexit.register(shutil.rmtree, BACKUPDB_DIRECTORY) diff --git a/tests/integration_test.py b/tests/integration_test.py new file mode 100644 index 0000000..3f7348e --- /dev/null +++ b/tests/integration_test.py @@ -0,0 +1,47 @@ +import tempfile +import shutil + +import dj_database_url +db_config = dj_database_url.config() +name = db_config['NAME'] +db_config['TEST_NAME'] = name +db_config['NAME'] = '_{}'.format(name) + +from django.conf import settings +settings.configure( + DEBUG=True, + DATABASES={'default': db_config}, + INSTALLED_APPS=('django.contrib.contenttypes', 'backupdb', ) +) + +from django.db import models +from django.test import SimpleTestCase +from django.test.utils import override_settings +from django.core import management +from django.contrib.contenttypes.models import ContentType + + +class BackupTest(SimpleTestCase): + + def setUp(self): + self.backup_dir = tempfile.mkdtemp('backupdb') + self._settings = override_settings(BACKUPDB_DIRECTORY=self.backup_dir) + self._settings.enable() + + def test_backupdb(self): + ct = ContentType.objects.create() + # There should be a default content + assert ContentType.objects.filter(pk=ct.pk).exists() + + management.call_command('backupdb') + + ContentType.objects.all().delete() + assert not ContentType.objects.filter(pk=ct.pk).exists() + + management.call_command('restoredb') + + assert ContentType.objects.filter(pk=ct.pk).exists() + + def tearDown(self): + self._settings.disable() + shutil.rmtree(self.backup_dir) diff --git a/tests/unit_test.py b/tests/unit_test.py new file mode 100644 index 0000000..e6e8ddc --- /dev/null +++ b/tests/unit_test.py @@ -0,0 +1,82 @@ +import tempfile +import shutil +import os +import unittest + +from backupdb.utils import get_latest_timestamped_file + + +class GetLatestFileTest(unittest.TestCase): + + def setUp(self): + self.directory = tempfile.mkdtemp(suffix='backupdb') + + + def tearDown(self): + shutil.rmtree(self.directory) + + def create_files(self, file_list): + for fname in file_list: + absolute_fname = os.path.join(self.directory, fname) + open(absolute_fname, 'w').close() + + def test_can_find_old_format(self): + self.create_files([ + 'default-2013-05-02-1367553089.sqlite.gz', + 'default-2013-06-06-1370570260.sqlite.gz', + 'default-2013-06-06-1370580510.sqlite.gz', + + 'default-2013-05-02-1367553089.mysql.gz', + 'default-2013-06-06-1370570260.mysql.gz', + 'default-2013-06-06-1370580510.mysql.gz', + + 'default-2013-05-02-1367553089.pgsql.gz', + 'default-2013-06-06-1370570260.pgsql.gz', + 'default-2013-06-06-1370580510.pgsql.gz', + ]) + + sqlite = get_latest_timestamped_file( + 'default', 'sqlite', directory=self.directory) + assert sqlite == 'default-2013-06-06-1370580510.sqlite.gz' + + + mysql = get_latest_timestamped_file( + 'default', 'mysql', directory=self.directory) + assert mysql == 'default-2013-06-06-1370580510.mysql.gz' + + postgresql = get_latest_timestamped_file( + 'default', 'pgsql', directory=self.directory) + assert postgresql == 'default-2013-06-06-1370580510.pgsql.gz' + + def test_can_find_new_format(self): + self.create_files([ + 'default-2015-04-06T00:00:00.000000.sqlite.gz', + 'default-2015-04-06T00:01:00.000000.sqlite.gz', + 'default-2015-04-06T01:00:00.000000.sqlite.gz', + ]) + + fname = get_latest_timestamped_file( + 'default', 'sqlite', directory=self.directory) + + assert fname == 'default-2015-04-06T01:00:00.000000.sqlite.gz' + + + self.create_files([ + 'default-2015-04-07T00:00:00.000000.sqlite.gz', + 'default-2015-05-06T00:00:00.000000.sqlite.gz', + 'default-2016-04-06T00:00:00.000000.sqlite.gz', + ]) + + fname = get_latest_timestamped_file( + 'default', 'sqlite', directory=self.directory) + + assert fname == 'default-2016-04-06T00:00:00.000000.sqlite.gz' + + def test_cant_find_file(self): + default = get_latest_timestamped_file( + '', '', directory=self.directory) + assert default is None + + mysql = get_latest_timestamped_file( + 'default', 'mysql', directory=self.directory) + assert mysql is None diff --git a/unit_tests_scratch/.gitkeep b/unit_tests_scratch/.gitkeep deleted file mode 100644 index e69de29..0000000 From 7548740721f61be19a21821cadf253f4514e1eed Mon Sep 17 00:00:00 2001 From: Antoine Catton Date: Tue, 7 Apr 2015 11:32:46 -0600 Subject: [PATCH 2/3] Add tox and integrate it to travis --- .gitignore | 1 + .travis.yml | 43 +++++++++++++++++++++++++++++++-------- tests/integration_test.py | 9 +++++--- tox.ini | 25 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index cf248bd..f91d379 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ test_backupdb_database backups/ .coverage .b_hook +/.tox/ diff --git a/.travis.yml b/.travis.yml index f29536d..483d5d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,45 @@ language: python python: + - "3.4" - "3.3" - "2.7" - "2.6" +services: + - mysql + - postgres env: - - DJANGO="Django==1.4" - - DJANGO="Django==1.5" - - DJANGO="Django==1.6" - - DJANGO="https://github.com/django/django/archive/1.7a2.tar.gz#egg=Django" + - DB=sqlite + - DB=mysql + - DB=postresql + + - DJANGO=1.4 + - DJANGO=1.6 + - DJANGO=1.7 + - DJANGO=1.8 matrix: exclude: + # Django 1.4 never supported Python 3 + - python: "3.3" + env: DJANGO="1.4" + - python: "3.4" + env: DJANGO="1.4" # Python 2.6 support has been dropped in Django 1.7 - python: "2.6" - env: DJANGO="https://github.com/django/django/archive/1.7a2.tar.gz#egg=Django" - # Support for python 3 was added in Django 1.5 - - python: "3.3" - env: DJANGO="Django==1.4" -install: pip install $DJANGO + env: DJANGO="1.7" + - python: "2.6" + env: DJANGO="1.8" +before_script: + - test [ $DB = mysql ] && mysql -e "create database if not exists travis_ci_test;" -uroot + - test [ $DB = postgres ] && psql -e "create database if not exists travis_ci_test;" -U postgres script: python setup.py test + # This needs to be smart :( + - export PYVER=$(python3 -V | sed -e 's/^Python \([0-9]\)\.\([0-9]\)\.[0-9]\+$/\1\2/') + - export DJVER=$(echo $DJANGO | sed -e 's/\.//g') + - echo PYVER=$PYVER + - echo DJVER=$DJVER + - export DBNAME=/travis_ci_test + - test [ $DB = sqlite ] && export DBNAME="/tmp${DBNAME}" + - test [ $DB = postgres ] && export DBUSER="postgres@" + - test [ $DB = mysql ] && export DBUSER="root@" + - export DATABASE_URL="${DB}://${DBUSER}/${DBNAME}" + - tox -e "py${PYVER}-dj${DJVER}-$DB" diff --git a/tests/integration_test.py b/tests/integration_test.py index 3f7348e..cdd16e5 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -3,9 +3,6 @@ import dj_database_url db_config = dj_database_url.config() -name = db_config['NAME'] -db_config['TEST_NAME'] = name -db_config['NAME'] = '_{}'.format(name) from django.conf import settings settings.configure( @@ -19,6 +16,10 @@ from django.test.utils import override_settings from django.core import management from django.contrib.contenttypes.models import ContentType +import django + +if hasattr(django, 'setup'): # Django > 1.6 + django.setup() # Setup the app registry class BackupTest(SimpleTestCase): @@ -29,6 +30,8 @@ def setUp(self): self._settings.enable() def test_backupdb(self): + management.call_command('syncdb') + ct = ContentType.objects.create() # There should be a default content assert ContentType.objects.filter(pk=ct.pk).exists() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..44b9ad0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +envlist = + py{26,27}-dj{14,16}-{sqlite,mysql,postgres} + py{33,34}-dj16-{sqlite,mysql,postgres} + py{27,33,34}-dj{17,18}-{sqlite,mysql,postres} + +[testenv] +basepython= + py26: python2.6 + py27: python2.7 + py33: python3.3 + py34: python3.4 + +deps= + pytest + dj-database-url + mysql: mysql-python + postgres: psycopg2 + + dj14: Django>=1.4,<1.5 + dj16: Django>=1.6,<1.7 + dj17: Django>=1.7,<1.8 + dj18: Django>=1.8,<1.9 + +commands=py.test {posargs} From fc7d478179549a5ce59e61824d9a2e1cc4acd240 Mon Sep 17 00:00:00 2001 From: Antoine Catton Date: Tue, 7 Apr 2015 13:45:19 -0600 Subject: [PATCH 3/3] Get the absolute path of the backup directory Thank you Piper Merriam for the suggestion. --- backupdb/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backupdb/settings.py b/backupdb/settings.py index 547845f..d34914c 100644 --- a/backupdb/settings.py +++ b/backupdb/settings.py @@ -1,3 +1,5 @@ +import os + from django.conf import settings from backupdb.backends.mysql import MySQLBackend @@ -16,4 +18,5 @@ def get_backup_directory(): - return getattr(settings, 'BACKUPDB_DIRECTORY', DEFAULT_BACKUP_DIR) + return os.path.abspath( + getattr(settings, 'BACKUPDB_DIRECTORY', DEFAULT_BACKUP_DIR))