From ee3f6cba9fdfba18606e84a15aa733211cf3901c Mon Sep 17 00:00:00 2001 From: karitra Date: Tue, 2 Jul 2019 02:27:21 +0300 Subject: [PATCH 1/6] feat: option to move restored data to origins --- eblob_kit.py | 292 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 211 insertions(+), 81 deletions(-) diff --git a/eblob_kit.py b/eblob_kit.py index 2a5603c..829b3cd 100755 --- a/eblob_kit.py +++ b/eblob_kit.py @@ -10,9 +10,11 @@ import logging import os import re +import shutil import struct import sys +from contextlib import contextmanager from datetime import datetime from datetime import timedelta @@ -20,6 +22,35 @@ import pyhash +INDEX_SUFFIX = '.index' +SORTED_INDEX_SUFFIX = INDEX_SUFFIX + '.sorted' +TO_REMOVE_SUFFIX = '.should_be_removed' + + +@contextmanager +def managed_blob(path, is_index_sorted=False): + """Wrap blob into context with implicit close.""" + try: + blob = None + blob = Blob.create(path, is_index_sorted) + yield blob + finally: + if blob is not None: + blob.close() + + +@contextmanager +def managed_index(path): + """Wrap index into context with implicit close.""" + try: + index = None + index = IndexFile.create(path) + yield index + finally: + if index is not None: + index.close() + + def dump_digest(verbosity, results_digest): """Dump report to console as JSON. @@ -348,7 +379,6 @@ def __init__(self, key, data_size, disk_size, flags, position): @staticmethod def from_raw(data): """Initialize from raw @data.""" - assert len(data) == DiskControl.size key = data[:DiskControl.key_size] @@ -407,9 +437,9 @@ class IndexFile(object): def __init__(self, path, mode='rb'): """Initialize IndexFile object again @path.""" - if path.endswith('.index.sorted'): + if path.endswith(SORTED_INDEX_SUFFIX): self.sorted = True - elif path.endswith('.index'): + elif path.endswith(INDEX_SUFFIX): self.sorted = False else: raise RuntimeError('{} is not index'.format(path)) @@ -433,6 +463,31 @@ def file(self): """Return file.""" return self._file + def close(self): + """Close underlying file.""" + self._file.close() + + @staticmethod + def move_back(from_index, to_index): + """Move underlying file(s) between specified pathes.""" + if not to_index.endswith(SORTED_INDEX_SUFFIX): + # Move possible sorted repeared index to not sorted original. + shutil.move(src=from_index, dst=to_index) + return + + if not from_index.endswith(SORTED_INDEX_SUFFIX): + # State when destination index is sorted, but repaired index isn't + # sorted. Store destination file with specified suffix. + temporary_name = to_index + TO_REMOVE_SUFFIX + + shutil.move(src=to_index, dst=temporary_name) + shutil.move(src=from_index, dst=to_index) + return + + # NOTE: Moving from sorted index to possibly unsorted. + # Don't mark destination as sroted for sake of simplicity. + shutil.move(src=from_index, dst=to_index) + def append(self, header): """Append header to index.""" self._file.write(header.raw_data) @@ -468,7 +523,7 @@ class DataFile(object): def __init__(self, path, mode='rb'): """Initialize DataFile object again @path.""" self.sorted = os.path.exists(path + '.data_is_sorted') and \ - os.path.exists(path + '.index.sorted') + os.path.exists(path + SORTED_INDEX_SUFFIX) self._file = open(path, mode) @property @@ -481,6 +536,15 @@ def file(self): """Return file.""" return self._file + def close(self): + """Close underlying file.""" + self._file.close() + + @staticmethod + def move_back(from_data, to_data): + """Move underlying file(s) between specified pathes.""" + shutil.move(src=from_data, dst=to_data) + def read_disk_control(self, position): """Read DiskControl at @offset.""" return DiskControl.from_raw(self.read(position, DiskControl.size)) @@ -525,25 +589,25 @@ class Blob(object): def __init__(self, path, mode='rb'): """Initialize Blob object against @path.""" - if os.path.exists(path + '.index.sorted'): - self._index_file = IndexFile(path + '.index.sorted', mode) - elif os.path.exists(path + '.index'): - self._index_file = IndexFile(path + '.index', mode) + if os.path.exists(path + SORTED_INDEX_SUFFIX): + self._index_file = IndexFile(path + SORTED_INDEX_SUFFIX, mode) + elif os.path.exists(path + INDEX_SUFFIX): + self._index_file = IndexFile(path + INDEX_SUFFIX, mode) else: raise IOError('Could not find index for {}'.format(path)) self._data_file = DataFile(path, mode) @staticmethod - def create(path, mark_index_sorted=False): + def create(path, is_index_sorted=False): """Create new Blob at @path. - NOTE: underlying files are truncuated if they are exist. + NOTE: underlying files are truncated if they are exist. """ - index_suffix = '.index.sorted' if mark_index_sorted else '.index' + index_suffix = SORTED_INDEX_SUFFIX if is_index_sorted else INDEX_SUFFIX create_mode = 'wb' - # Index is checked for existance on Blob creation, so we should create it + # Index is checked for existence on Blob creation, so we should create it # beforehand, but data file would be created within Blob constructor. open(path + index_suffix, create_mode).close() @@ -559,6 +623,25 @@ def data(self): """Return data file.""" return self._data_file + def get_index_data_path_tuple(self): + """Get underlying files paths.""" + return self.index.path, self.data.path + + def close(self): + """Closes underlying files.""" + self._index_file.close() + self._data_file.close() + + @staticmethod + def move_back(from_index, from_data, to_index, to_data): + """Move underlying file(s) between specified pathes.""" + logging.info('Moving blob files back: from_index="%s", from_data="%s", to_index="%s", to_data="%s"', + from_index, from_data, + to_index, to_data) + + DataFile.move_back(from_data=from_data, to_data=to_data) + IndexFile.move_back(from_index=from_index, to_index=to_index) + def _murmur_chunk(self, chunk): """Apply murmurhash to chunk and return raw result.""" chunk_size = 4096 @@ -780,7 +863,6 @@ def __init__(self, path): self._valid = True self._index_headers = [] - self._stat = BlobRepairerStat() @property @@ -915,22 +997,26 @@ def check_index(self, fast): logging.info('Checking index: %s', self._blob.index.path) for header in self._blob.index: + if fast and not self._valid: # return if it is fast-check and there was an error return + if self.check_header(header): self._index_headers.append(header) + if prev_key and self._blob.index.sorted: if prev_key > header.key: self._valid = False self._blob.index.sorted = False self._stat.index_order_error = True + prev_key = header.key else: self._stat.index_malformed_headers_keys.add(header.hex_key) self._stat.index_malformed_headers += 1 - self._valid = False + logging.debug('Index malformed: hex_key="%s"', header.hex_key) except EOFError as exc: logging.error('Failed to read header path %s, error %s. Skip other headers in index', self._blob.index.path, @@ -1022,6 +1108,10 @@ def check(self, verify_csum, fast): self.resolve_mispositioned_record(header_idx, position, valid_headers) if position < index_header.position: + logging.warning('Malformed index_header: position="%s", expected_position="%s"', + position, + index_header.position) + self._valid = False self.check_hole(position, index_header.position) @@ -1041,6 +1131,7 @@ def check(self, verify_csum, fast): self._stat.corrupted_data_headers += 1 self._stat.corrupted_data_headers_keys.add(data_header.hex_key) self._stat.corrupted_data_headers_size += index_header.disk_size + logging.warn('Corrupted data header: header="%s"', index_header) self._valid = False else: valid_headers.append(index_header) @@ -1052,6 +1143,10 @@ def check(self, verify_csum, fast): position = index_header.position + index_header.disk_size if position < len(self._blob.data): + logging.warn('Last position less then blob size: position="%s", blob_size="%s"', + position, + len(self._blob.data)) + self._valid = False self.check_hole(position, len(self._blob.data)) @@ -1060,34 +1155,41 @@ def check(self, verify_csum, fast): return self._valid @staticmethod - def recover_index(data, destination, overwrite=False): + def recover_index(data, destination, overwrite=False, move_back=False): """Recover index from data.""" basename = os.path.basename(data.path) - index_path = os.path.join(destination, basename + '.index') + index_path = os.path.join(destination, basename + INDEX_SUFFIX) - if not is_destination_writable(data.path + '.index', index_path, overwrite): + if not is_destination_writable(data.path + INDEX_SUFFIX, index_path, overwrite): raise RuntimeError("can't recover to already existing index file: {}".format(index_path)) - index = IndexFile.create(index_path) - logging.info('Recovering index %s -> %s', data.path, index_path) - for header in data: - if header: - index.append(header) - continue - - offset = data.file.tell() - DiskControl.size - logging.error('I have found broken header at offset %s: %s', offset, header) - logging.error('This record can not be skipped, so I break the recovering. ' - 'You can use %s as an index for %s but it does not include ' - 'records after %s offset', - index.path, - data.path, - offset) - break - - def recover_blob(self, destination, overwrite=False): + with managed_index(index_path) as index: + + destination_index_path = index.path + + for header in data: + if header: + index.append(header) + continue + + offset = data.file.tell() - DiskControl.size + logging.error('I have found broken header at offset %s: %s', offset, header) + logging.error('This record can not be skipped, so I break the recovering. ' + 'You can use %s as an index for %s but it does not include ' + 'records after %s offset', + index.path, + data.path, + offset) + break + + logging.info('Recover_index post processing: move_back="%s"', move_back) + + if move_back: + IndexFile.move_back(from_index=destination_index_path, to_index=data.path + INDEX_SUFFIX) + + def recover_blob(self, destination, overwrite=False, move_back=False, check_result=False): """Recover blob from data.""" basename = os.path.basename(self._blob.data.path) blob_path = os.path.join(destination, basename) @@ -1095,30 +1197,37 @@ def recover_blob(self, destination, overwrite=False): if not is_destination_writable(self._blob.data.path, blob_path, overwrite): raise RuntimeError("can't recover to already existing blob file: {}".format(blob_path)) - blob = Blob.create(path=blob_path) - copied_records = 0 removed_records = 0 skipped_records = 0 logging.info('Recovering blob %s -> %s', self._blob.data.path, blob_path) - for header in self._blob.data: - if not header: - skipped_records += 1 - logging.error('I have faced with broken record which I have to skip.') - elif header.flags.removed: - removed_records += 1 - else: - copy_record(self._blob, blob, header) - copied_records += 1 + with managed_blob(blob_path, self._blob.index.sorted) as blob: + + destination_index_path, destination_data_path = blob.get_index_data_path_tuple() + + for header in self._blob.data: + if not header: + skipped_records += 1 + logging.error('I have faced with broken record which I have to skip.') + elif header.flags.removed: + removed_records += 1 + else: + copy_record(self._blob, blob, header) + copied_records += 1 logging.info('I have copied %s records, skipped %s and removed %s records', copied_records, skipped_records, removed_records) - def copy_valid_records(self, destination, overwrite=False): + if move_back: + self._blob.close() + self._blob.move_from(from_index=destination_index_path, + from_data=destination_data_path) + + def copy_valid_records(self, destination, overwrite=False, move_back=False): """Recover blob by copying only valid records from blob.""" basename = os.path.basename(self._blob.data.path) blob_path = os.path.join(destination, basename) @@ -1126,26 +1235,34 @@ def copy_valid_records(self, destination, overwrite=False): if not is_destination_writable(self._blob.data.path, blob_path, overwrite): raise RuntimeError("can't copy valid records to already existing blob file: {}".format(blob_path)) - blob = Blob.create(blob_path) + self._index_headers += self._stat.data_recoverable_headers + logging.info('Recovering blob %s -> %s', self._blob.data.path, blob_path) copied_records = 0 copied_size = 0 - self._index_headers += self._stat.data_recoverable_headers - logging.info('Recovering blob %s -> %s', self._blob.data.path, blob_path) + with managed_blob(blob_path) as dst_blob: - for header in self._index_headers: - copy_record(self._blob, blob, header) - copied_records += 1 - copied_size += header.disk_size + destination_index_path, destination_blob_path = dst_blob.get_index_data_path_tuple() - logging.info('I have copied %s (%s) records %s -> %s ', - copied_records, - sizeof_fmt(copied_size), - self._blob.data.path, - blob_path) + for header in self._index_headers: + copy_record(self._blob, dst_blob, header) + copied_records += 1 + copied_size += header.disk_size + + logging.info('I have copied %s (%s) records %s -> %s ', + copied_records, + sizeof_fmt(copied_size), + self._blob.data.path, + blob_path) - def fix(self, destination, noprompt, overwrite=False): + if move_back: + # Both destination and source blobs should be closed at this point. + self._blob.close() + self._blob.move_from(from_index=destination_index_path, + from_data=destination_blob_path) + + def fix(self, destination, noprompt, overwrite=False, move_back=False): """Check blob's data & index and try to fix them if they are broken. TODO(karapuz): remove all interactive user interaction. @@ -1155,6 +1272,7 @@ def fix(self, destination, noprompt, overwrite=False): self.print_check_report() if self._valid: + logging.info('All records are valid in %s', self._blob.data.path) return if (not self._index_headers and @@ -1162,26 +1280,27 @@ def fix(self, destination, noprompt, overwrite=False): not self._stat.index_uncommitted_headers): if noprompt: - self.recover_blob(destination, overwrite=overwrite) + self.recover_blob(destination, overwrite=overwrite, move_back=move_back) elif click.confirm('There is no valid header in {}. ' 'Should I try to recover index from {}?' .format(self._blob.index.path, self._blob.data.path), default=True): - self.recover_index(self._blob.data, destination, overwrite=overwrite) + self.recover_index(self._blob.data, destination, overwrite=overwrite, move_back=move_back) elif click.confirm('Should I try to recover both index and data from {}?' .format(self._blob.data.path), default=True): - self.recover_blob(destination, overwrite=overwrite) + self.recover_blob(destination, overwrite=overwrite, move_back=move_back) else: if not self._index_headers: logging.error('Nothing can be recovered from %s, so it should be removed', self._blob.data.path) - filname = '{}.should_be_removed'.format( - os.path.join(destination, os.path.basename(self._blob.data.path))) - with open(filname, 'wb'): + filename = '{}{}'.format( + os.path.join(destination, os.path.basename(self._blob.data.path)), + TO_REMOVE_SUFFIX) + with open(filename, 'wb'): pass elif noprompt or click.confirm('Should I repair {}?'.format(self._blob.data.path), default=True): - self.copy_valid_records(destination, overwrite=overwrite) + self.copy_valid_records(destination, overwrite=overwrite, move_back=move_back) def find_duplicates(blobs): @@ -1190,10 +1309,10 @@ def find_duplicates(blobs): duplicates = {} for blob in blobs: index_file = None - if os.path.exists(blob + '.index.sorted'): - index_file = IndexFile(blob + '.index.sorted') - elif os.path.exists(blob + '.index'): - index_file = IndexFile(blob + '.index') + if os.path.exists(blob + SORTED_INDEX_SUFFIX): + index_file = IndexFile(blob + SORTED_INDEX_SUFFIX) + elif os.path.exists(blob + INDEX_SUFFIX): + index_file = IndexFile(blob + INDEX_SUFFIX) else: raise IOError('Could not find index for {}'.format(blob)) @@ -1316,10 +1435,10 @@ def restore_keys(blobs, keys, short, checksum_type): for blob_path in blobs: blob_idx = int(blob_path[blob_path.find('.') + 1:]) - if os.path.exists(blob_path + '.index.sorted'): - index_file = IndexFile(blob_path + '.index.sorted') - elif os.path.exists(blob_path + '.index'): - index_file = IndexFile(blob_path + '.index') + if os.path.exists(blob_path + SORTED_INDEX_SUFFIX): + index_file = IndexFile(blob_path + SORTED_INDEX_SUFFIX) + elif os.path.exists(blob_path + INDEX_SUFFIX): + index_file = IndexFile(blob_path + INDEX_SUFFIX) else: raise IOError('Could not find index for {}'.format(blob_path)) @@ -1442,7 +1561,7 @@ def list_data_command(path): @click.pass_context def list_command(ctx, path): """List data or index file specified by @PATH.""" - if path.endswith('.index') or path.endswith('.index.sorted'): + if path.endswith(INDEX_SUFFIX) or path.endswith(SORTED_INDEX_SUFFIX): ctx.invoke(list_index_command, path=path) else: ctx.invoke(list_data_command, path=path) @@ -1500,8 +1619,10 @@ def fix_index_command(path, destination, overwrite): help='Assume Yes to all queries and do not prompt') @click.option('-o', '--overwrite', is_flag=True, default=False, help='Overwrite destination files') +@click.option('-m', '--move-back', is_flag=True, default=False, + help='Move restored files from destination back to original path') @click.pass_context -def fix_blob_command(ctx, path, destination, noprompt, overwrite): +def fix_blob_command(ctx, path, destination, noprompt, overwrite, move_back): """Fix one blob @PATH. TODO(karapuz): get rid of noprompt and interactivity. @@ -1516,7 +1637,7 @@ def fix_blob_command(ctx, path, destination, noprompt, overwrite): os.mkdir(destination) blob_repairer = BlobRepairer(path) - blob_repairer.fix(destination, noprompt, overwrite) + blob_repairer.fix(destination, noprompt, overwrite, move_back) # FIX_BLOB_STANDALONE - means that fix_blob_command not called from another subcommand. # TODO(karapuz): refactor ctx fields into well defined constants. @@ -1537,8 +1658,10 @@ def fix_blob_command(ctx, path, destination, noprompt, overwrite): "will be switched on when verbosity option not set") @click.option('-o', '--overwrite', is_flag=True, default=False, help='Overwrite destination files') +@click.option('-m', '--move-back', is_flag=True, default=False, + help='Move restored files from destination back to original path') @click.pass_context -def fix_command(ctx, path, destination, noprompt, overwrite): +def fix_command(ctx, path, destination, noprompt, overwrite, move_back): """Fix blobs @PATH.""" verbosity = ctx.obj.get('VERBOSITY', Verbosity.JSON) @@ -1549,7 +1672,12 @@ def fix_command(ctx, path, destination, noprompt, overwrite): for blob in files(path): try: - ctx.invoke(fix_blob_command, path=blob, destination=destination, noprompt=noprompt, overwrite=overwrite) + ctx.invoke(fix_blob_command, + path=blob, + destination=destination, + noprompt=noprompt, + overwrite=overwrite, + move_back=move_back) except Exception as exc: logging.error('Failed to fix %s: %s', blob, exc) raise @@ -1561,6 +1689,7 @@ def fix_command(ctx, path, destination, noprompt, overwrite): dump_digest(verbosity, results_digest) dump_to_file(ctx.obj.get(JSON_OUTPUT), results) + @cli.command(name='find_duplicates') @click.argument('path') @click.pass_context @@ -1582,7 +1711,7 @@ def remove_duplicates_command(ctx, path): @cli.command(name='restore_keys') @click.argument('path') -@click.option('-k', '--keys', 'keys_path', prompt='Where should I found keys to resotre', +@click.option('-k', '--keys', 'keys_path', prompt='Where should I found keys to restore', help='k for keys to restore') @click.option('--short', is_flag=True) @click.option('--checksum-type', type=click.Choice([ChecksumTypes.SHA512, ChecksumTypes.CHUNKED]), default=None, @@ -1602,5 +1731,6 @@ def main(): """Main function.""" cli(obj={}) + if __name__ == '__main__': main() From a53e8578c27c1e60d0c0b91876bc90d4a08d8215 Mon Sep 17 00:00:00 2001 From: karitra Date: Tue, 2 Jul 2019 17:31:10 +0300 Subject: [PATCH 2/6] feat: check_result flag to verify fix results --- eblob_kit.py | 63 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/eblob_kit.py b/eblob_kit.py index 829b3cd..3ad034a 100755 --- a/eblob_kit.py +++ b/eblob_kit.py @@ -1222,12 +1222,22 @@ def recover_blob(self, destination, overwrite=False, move_back=False, check_resu skipped_records, removed_records) + logging.info('Post recover blob processing: check_result="%s", move_back="%s"', + check_result, + move_back) + + if check_result and not BlobRepairer(blob_path).check(verify_csum=True, fast=False): + logging.error("check of fixed blob failed: path='%s'", blob_path) + raise RuntimeError('restored blob check has been failed') + if move_back: self._blob.close() - self._blob.move_from(from_index=destination_index_path, - from_data=destination_data_path) + self._blob.move_back(from_index=destination_index_path, + from_data=destination_data_path, + to_index=self._blob.index.path, + to_data=self._blob.data.path) - def copy_valid_records(self, destination, overwrite=False, move_back=False): + def copy_valid_records(self, destination, overwrite=False, move_back=False, check_result=False): """Recover blob by copying only valid records from blob.""" basename = os.path.basename(self._blob.data.path) blob_path = os.path.join(destination, basename) @@ -1256,13 +1266,21 @@ def copy_valid_records(self, destination, overwrite=False, move_back=False): self._blob.data.path, blob_path) + if check_result and not BlobRepairer(blob_path).check(verify_csum=True, fast=False): + logging.error("check of fixed blob failed: path='%s'", blob_path) + raise RuntimeError('restored blob check has been failed') + if move_back: # Both destination and source blobs should be closed at this point. self._blob.close() - self._blob.move_from(from_index=destination_index_path, - from_data=destination_blob_path) + to_index, to_data = self._blob.get_index_data_path_tuple() + + self._blob.move_back(from_index=destination_index_path, + from_data=destination_blob_path, + to_index=to_index, + to_data=to_data) - def fix(self, destination, noprompt, overwrite=False, move_back=False): + def fix(self, destination, noprompt, overwrite=False, move_back=False, check_result=False): """Check blob's data & index and try to fix them if they are broken. TODO(karapuz): remove all interactive user interaction. @@ -1280,16 +1298,25 @@ def fix(self, destination, noprompt, overwrite=False, move_back=False): not self._stat.index_uncommitted_headers): if noprompt: - self.recover_blob(destination, overwrite=overwrite, move_back=move_back) + self.recover_blob(destination, + overwrite=overwrite, + move_back=move_back, + check_result=check_result) elif click.confirm('There is no valid header in {}. ' 'Should I try to recover index from {}?' .format(self._blob.index.path, self._blob.data.path), default=True): - self.recover_index(self._blob.data, destination, overwrite=overwrite, move_back=move_back) + self.recover_index(self._blob.data, + destination, + overwrite=overwrite, + move_back=move_back) elif click.confirm('Should I try to recover both index and data from {}?' .format(self._blob.data.path), default=True): - self.recover_blob(destination, overwrite=overwrite, move_back=move_back) + self.recover_blob(destination, + overwrite=overwrite, + move_back=move_back, + check_result=check_result) else: if not self._index_headers: logging.error('Nothing can be recovered from %s, so it should be removed', self._blob.data.path) @@ -1300,7 +1327,10 @@ def fix(self, destination, noprompt, overwrite=False, move_back=False): pass elif noprompt or click.confirm('Should I repair {}?'.format(self._blob.data.path), default=True): - self.copy_valid_records(destination, overwrite=overwrite, move_back=move_back) + self.copy_valid_records(destination, + overwrite=overwrite, + move_back=move_back, + check_result=check_result) def find_duplicates(blobs): @@ -1621,8 +1651,10 @@ def fix_index_command(path, destination, overwrite): help='Overwrite destination files') @click.option('-m', '--move-back', is_flag=True, default=False, help='Move restored files from destination back to original path') +@click.option('-c', '--check-result', is_flag=True, default=False, + help='Run check on resulting blob') @click.pass_context -def fix_blob_command(ctx, path, destination, noprompt, overwrite, move_back): +def fix_blob_command(ctx, path, destination, noprompt, overwrite, move_back, check_result): """Fix one blob @PATH. TODO(karapuz): get rid of noprompt and interactivity. @@ -1637,7 +1669,7 @@ def fix_blob_command(ctx, path, destination, noprompt, overwrite, move_back): os.mkdir(destination) blob_repairer = BlobRepairer(path) - blob_repairer.fix(destination, noprompt, overwrite, move_back) + blob_repairer.fix(destination, noprompt, overwrite, move_back, check_result) # FIX_BLOB_STANDALONE - means that fix_blob_command not called from another subcommand. # TODO(karapuz): refactor ctx fields into well defined constants. @@ -1660,8 +1692,10 @@ def fix_blob_command(ctx, path, destination, noprompt, overwrite, move_back): help='Overwrite destination files') @click.option('-m', '--move-back', is_flag=True, default=False, help='Move restored files from destination back to original path') +@click.option('-c', '--check-result', is_flag=True, default=False, + help='Run check on resulting blob') @click.pass_context -def fix_command(ctx, path, destination, noprompt, overwrite, move_back): +def fix_command(ctx, path, destination, noprompt, overwrite, move_back, check_result): """Fix blobs @PATH.""" verbosity = ctx.obj.get('VERBOSITY', Verbosity.JSON) @@ -1677,7 +1711,8 @@ def fix_command(ctx, path, destination, noprompt, overwrite, move_back): destination=destination, noprompt=noprompt, overwrite=overwrite, - move_back=move_back) + move_back=move_back, + check_result=check_result) except Exception as exc: logging.error('Failed to fix %s: %s', blob, exc) raise From 037c6c3ef30af356140780ca469d01e964cb83e5 Mon Sep 17 00:00:00 2001 From: karitra Date: Tue, 2 Jul 2019 13:38:01 +0300 Subject: [PATCH 3/6] test: move_back option support --- tests/test_blob.py | 10 ++++++++++ tests/test_blob_repairer.py | 14 ++++++++++---- tests/test_index_file.py | 10 ++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/test_blob.py b/tests/test_blob.py index 5ce1919..fc4de12 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -33,3 +33,13 @@ def test_create_valid(mocked_open, mocked_exists): def test_create_incorrect_path(): """Test Blob.create static method with incorrect path.""" eblob_kit.Blob.create('') + + +@mock.patch('eblob_kit.Blob', autospec=True) +def test_blob_managed_close(mocked_blob): + """Test blob context manager.""" + mocked_blob.create.return_value = mocked_blob + with eblob_kit.managed_blob(''): + pass + + assert mocked_blob.close.call_count == 1 diff --git a/tests/test_blob_repairer.py b/tests/test_blob_repairer.py index 24f831d..e06e47b 100644 --- a/tests/test_blob_repairer.py +++ b/tests/test_blob_repairer.py @@ -14,7 +14,6 @@ """ import mock import pytest -import sys from eblob_kit import BlobRepairer from eblob_kit import EllipticsHeader @@ -418,7 +417,7 @@ def test_check_index_non_valid(_mocked_exists, @mock.patch('eblob_kit.is_destination_writable', return_value=True) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) @mock.patch('eblob_kit.Blob', autospec=True) -def test_fix_destination_writable(_mocked_blob, +def test_fix_destination_writable(mocked_blob, _mocked_open, _mocked_is_writable, callee): @@ -427,6 +426,9 @@ def test_fix_destination_writable(_mocked_blob, Checks for `copy_valid_records`, `recover_index` and `recover_blob`. """ + mocked_blob.create.return_value = mocked_blob + mocked_blob.get_index_data_path_tuple.return_value = (None, None) + blob_repairer = BlobRepairer('.') if callee == BlobRepairer.recover_index: @@ -477,10 +479,14 @@ def test_fix(mocker): type(index_file_class.return_value).sorted = mocker.PropertyMock(return_value=False) # DataFile mock - data_file_class = mocker.patch('eblob_kit.DataFile', autospec=True) + mocker.patch('eblob_kit.DataFile', autospec=True) # Blob mock - mocker.patch('eblob_kit.Blob.create', ) + mocked_blob = mock.Mock() + mocked_blob.create.return_value = mocked_blob + mocked_blob.get_index_data_path_tuple.return_value = (None, None) + + mocker.patch('eblob_kit.Blob.create', return_value=mocked_blob) mocker.patch('eblob_kit.is_destination_writable', return_value=True) mocker.patch('os.path.exists', return_value=True) diff --git a/tests/test_index_file.py b/tests/test_index_file.py index 54c64fe..ba3ebb3 100644 --- a/tests/test_index_file.py +++ b/tests/test_index_file.py @@ -28,3 +28,13 @@ def test_create_invalid_path(): def test_create_incorrect_path(): """Test IndexFile.create static method with incorrect path.""" eblob_kit.IndexFile.create('some/path/to/index') + + +@mock.patch('eblob_kit.IndexFile', autospec=True) +def test_index_managed_close(mocked_index): + """Test index context manager.""" + mocked_index.create.return_value = mocked_index + with eblob_kit.managed_index(''): + pass + + mocked_index.create.return_value = mocked_index From d238ebace975c9b557eb640d6e45b60505542bc5 Mon Sep 17 00:00:00 2001 From: karitra Date: Wed, 3 Jul 2019 15:37:30 +0300 Subject: [PATCH 4/6] refact: post review update 1, plus implement move and remove as methods --- eblob_kit.py | 323 +++++++++++++++++++++++++----------- tests/test_blob_repairer.py | 6 +- 2 files changed, 227 insertions(+), 102 deletions(-) diff --git a/eblob_kit.py b/eblob_kit.py index 3ad034a..f57b92e 100755 --- a/eblob_kit.py +++ b/eblob_kit.py @@ -23,10 +23,26 @@ INDEX_SUFFIX = '.index' -SORTED_INDEX_SUFFIX = INDEX_SUFFIX + '.sorted' +SORTED_INDEX_MARK = '.sorted' +SORTED_INDEX_SUFFIX = INDEX_SUFFIX + SORTED_INDEX_MARK + +SORTED_DATA_SUFFIX = '.data_is_sorted' TO_REMOVE_SUFFIX = '.should_be_removed' +def is_sorted(lst, key=lambda x: x): + """Check whether the sequence is sorted. + + Source: https://stackoverflow.com/a/3755153 + + """ + for i, el in enumerate(lst[1:]): + if key(el) < key(lst[i]): # i is the index of the previous element + return False + + return True + + @contextmanager def managed_blob(path, is_index_sorted=False): """Wrap blob into context with implicit close.""" @@ -467,26 +483,36 @@ def close(self): """Close underlying file.""" self._file.close() - @staticmethod - def move_back(from_index, to_index): - """Move underlying file(s) between specified pathes.""" - if not to_index.endswith(SORTED_INDEX_SUFFIX): - # Move possible sorted repeared index to not sorted original. - shutil.move(src=from_index, dst=to_index) - return + def remove(self): + """Remove underlying file(s).""" + if not self._file.closed: + self.close() - if not from_index.endswith(SORTED_INDEX_SUFFIX): - # State when destination index is sorted, but repaired index isn't - # sorted. Store destination file with specified suffix. - temporary_name = to_index + TO_REMOVE_SUFFIX + current_path = os.path.abspath(self._file.name) + logging.info('Removing index file: filename="%s"', current_path) + os.remove(current_path) - shutil.move(src=to_index, dst=temporary_name) - shutil.move(src=from_index, dst=to_index) - return + def move_to(self, remote_folder, reopen_mode='ab+'): + """Move underlying file(s) to remote_folder.""" + if not os.path.isdir(remote_folder): + raise ValueError('not a folder: "{}"'.format(remote_folder)) + + should_reopen = False + if not self._file.closed: + should_reopen = True + self.close() + + current_path = self.file.name + logging.debug('Moving index file: from="%s", remote_folder="%s"', + current_path, remote_folder) - # NOTE: Moving from sorted index to possibly unsorted. - # Don't mark destination as sroted for sake of simplicity. - shutil.move(src=from_index, dst=to_index) + shutil.move(src=current_path, dst=remote_folder) + + if should_reopen: + remote_path = os.path.basename(current_path) + remote_path = os.path.join(remote_folder, remote_path) + + self._file = open(remote_path, mode=reopen_mode) def append(self, header): """Append header to index.""" @@ -522,7 +548,7 @@ class DataFile(object): def __init__(self, path, mode='rb'): """Initialize DataFile object again @path.""" - self.sorted = os.path.exists(path + '.data_is_sorted') and \ + self.sorted = os.path.exists(path + SORTED_DATA_SUFFIX) and \ os.path.exists(path + SORTED_INDEX_SUFFIX) self._file = open(path, mode) @@ -540,10 +566,44 @@ def close(self): """Close underlying file.""" self._file.close() - @staticmethod - def move_back(from_data, to_data): - """Move underlying file(s) between specified pathes.""" - shutil.move(src=from_data, dst=to_data) + def remove(self): + if not self._file.closed: + self.close() + + current_path = os.path.abspath(self._file.name) + logging.info('Removing data file: filename="%s"', current_path) + os.remove(current_path) + + current_path_sorted_mark = current_path + SORTED_DATA_SUFFIX + if self.sorted and os.path.exists(current_path_sorted_mark): + logging.info('Removing sorted mark: filename="%s"', current_path_sorted_mark) + os.remove(current_path_sorted_mark) + + def move_to(self, remote_folder, reopen_mode='ab+'): + """Move underlying files to remote folder.""" + if not os.path.isdir(remote_folder): + raise ValueError('not a folder: "{}"'.format(remote_folder)) + + should_reopen = False + if not self._file.closed: + should_reopen = True + self._file.close() + + current_path = self.file.name + logging.debug('Moving data file: from="%s", remote_folder="%s"', current_path, remote_folder) + shutil.move(src=current_path, dst=remote_folder) + + if self.sorted: + current_path_sorted_mark = current_path + SORTED_DATA_SUFFIX + logging.debug('Moving data file sorted mark: from="%s", remote_folder="%s"', + current_path_sorted_mark, remote_folder) + shutil.move(src=current_path_sorted_mark, dst=remote_folder) + + if should_reopen: + remote_path = os.path.basename(current_path) + remote_path = os.path.join(remote_folder, remote_path) + + self._file = open(remote_path, mode=reopen_mode) def read_disk_control(self, position): """Read DiskControl at @offset.""" @@ -599,18 +659,22 @@ def __init__(self, path, mode='rb'): self._data_file = DataFile(path, mode) @staticmethod - def create(path, is_index_sorted=False): + def create(path, is_blob_sorted=False): """Create new Blob at @path. NOTE: underlying files are truncated if they are exist. """ - index_suffix = SORTED_INDEX_SUFFIX if is_index_sorted else INDEX_SUFFIX + index_suffix = SORTED_INDEX_SUFFIX if is_blob_sorted else INDEX_SUFFIX create_mode = 'wb' # Index is checked for existence on Blob creation, so we should create it # beforehand, but data file would be created within Blob constructor. open(path + index_suffix, create_mode).close() + if is_blob_sorted: + # Create sorted marker for data blob + open(path + SORTED_DATA_SUFFIX, create_mode).close() + return Blob(path=path, mode=create_mode) @property @@ -618,29 +682,33 @@ def index(self): """Return index file.""" return self._index_file + @index.setter + def index(self, new_index): + """Set provided index structure.""" + self._index_file.close() + self._index_file = new_index + @property def data(self): """Return data file.""" return self._data_file - def get_index_data_path_tuple(self): - """Get underlying files paths.""" - return self.index.path, self.data.path - def close(self): """Closes underlying files.""" self._index_file.close() self._data_file.close() - @staticmethod - def move_back(from_index, from_data, to_index, to_data): - """Move underlying file(s) between specified pathes.""" - logging.info('Moving blob files back: from_index="%s", from_data="%s", to_index="%s", to_data="%s"', - from_index, from_data, - to_index, to_data) + def remove(self): + """Remove underlying file(s).""" + self._index_file.remove() + self._data_file.remove() + + def move_to(self, remote_folder): + """Move underlying file(s) to remote folder.""" + logging.info('Moving blob files back: remote_folder="%s"', remote_folder) - DataFile.move_back(from_data=from_data, to_data=to_data) - IndexFile.move_back(from_index=from_index, to_index=to_index) + self._index_file.move_to(remote_folder) + self._data_file.move_to(remote_folder) def _murmur_chunk(self, chunk): """Apply murmurhash to chunk and return raw result.""" @@ -859,12 +927,16 @@ class BlobRepairer(object): def __init__(self, path): """Initialize BlobRepairer for blob at @path.""" + self._blob = Blob(path) + self._valid = True self._index_headers = [] self._stat = BlobRepairerStat() + self._sorted_index_headers = False + @property def stat(self): """Return BlobRepairerStat object.""" @@ -878,6 +950,11 @@ def valid(self): """ return self._valid + @property + def blob(self): + """Get target blob.""" + return self._blob + def check_header(self, header): """Check header correctness.""" if header.disk_size == 0: @@ -1150,28 +1227,35 @@ def check(self, verify_csum, fast): self._valid = False self.check_hole(position, len(self._blob.data)) + self._sorted_index_headers = is_sorted(lst=valid_headers, key=lambda h: h.key) self._index_headers = valid_headers return self._valid @staticmethod - def recover_index(data, destination, overwrite=False, move_back=False): + def recover_index(data, destination, overwrite=False, move_back=False, check_result=False): """Recover index from data.""" - basename = os.path.basename(data.path) - index_path = os.path.join(destination, basename + INDEX_SUFFIX) - if not is_destination_writable(data.path + INDEX_SUFFIX, index_path, overwrite): - raise RuntimeError("can't recover to already existing index file: {}".format(index_path)) + # It is assumed that original index could be absent, therefore it is + # marked here without '.sorted' suffix, so later on move_back stage, original + # 'sorted' index, if it exist, will be not removed. + source_index_path = os.path.abspath(data.path + INDEX_SUFFIX) + + destination_index_basename = os.path.basename(source_index_path) + destination_index_path = os.path.join(destination, destination_index_basename) - logging.info('Recovering index %s -> %s', data.path, index_path) + if not is_destination_writable(source_index_path, destination_index_path, overwrite): + raise RuntimeError( + "can't recover to already existing index file: {}".format(destination_index_path) + ) - with managed_index(index_path) as index: + logging.info('Recovering index from data %s -> %s', data.path, destination_index_path) - destination_index_path = index.path + with managed_index(destination_index_path) as destination_index: for header in data: if header: - index.append(header) + destination_index.append(header) continue offset = data.file.tell() - DiskControl.size @@ -1179,33 +1263,65 @@ def recover_index(data, destination, overwrite=False, move_back=False): logging.error('This record can not be skipped, so I break the recovering. ' 'You can use %s as an index for %s but it does not include ' 'records after %s offset', - index.path, + destination_index.path, data.path, offset) + break - logging.info('Recover_index post processing: move_back="%s"', move_back) + logging.info('Recover_index post processing: move_back="%s", check_result="%s"', + move_back, + check_result) + + if check_result: + # NOTE: internally blob opened with underlying index file, but open + # would failed if there is no index file for some reason. + repairer = BlobRepairer(data.path) + + try: + repairer.blob.index = IndexFile(destination_index.path) + repairer.check_index(fast=False) + finally: + repairer.blob.close() - if move_back: - IndexFile.move_back(from_index=destination_index_path, to_index=data.path + INDEX_SUFFIX) + if not repairer.valid: + logging.error("Check of recovered index failed: path='%s'", + destination_index.path) + raise RuntimeError('restored index check has been failed') + + if move_back: + source_folder = os.path.dirname(source_index_path) + + logging.info('Replacing index: from="%s", remote_folder="%s"', + destination_index.path, source_folder) + + try: + # NOTE: warning, all index data for `blob` would be deleted. + source_index = IndexFile(source_index_path) + source_index.remove() + except EnvironmentError as e: + logging.warn("Remove of index file has been failed: error='%s', path='%s'", + e, source_index_path) + + destination_index.move_to(source_folder) def recover_blob(self, destination, overwrite=False, move_back=False, check_result=False): """Recover blob from data.""" - basename = os.path.basename(self._blob.data.path) - blob_path = os.path.join(destination, basename) - if not is_destination_writable(self._blob.data.path, blob_path, overwrite): - raise RuntimeError("can't recover to already existing blob file: {}".format(blob_path)) + source_blob_path = self._blob.data.path + basename = os.path.basename(source_blob_path) + destination_blob_path = os.path.join(destination, basename) + + if not is_destination_writable(source_blob_path, destination_blob_path, overwrite): + raise RuntimeError("can't recover to already existing blob file: {}".format(destination_blob_path)) copied_records = 0 removed_records = 0 skipped_records = 0 - logging.info('Recovering blob %s -> %s', self._blob.data.path, blob_path) - - with managed_blob(blob_path, self._blob.index.sorted) as blob: + logging.info('Recovering blob %s -> %s', source_blob_path, destination_blob_path) - destination_index_path, destination_data_path = blob.get_index_data_path_tuple() + with managed_blob(destination_blob_path, self._sorted_index_headers) as destination_blob: for header in self._blob.data: if not header: @@ -1214,71 +1330,75 @@ def recover_blob(self, destination, overwrite=False, move_back=False, check_resu elif header.flags.removed: removed_records += 1 else: - copy_record(self._blob, blob, header) + copy_record(self._blob, destination_blob, header) copied_records += 1 - logging.info('I have copied %s records, skipped %s and removed %s records', - copied_records, - skipped_records, - removed_records) + logging.info('I have copied %s records, skipped %s and removed %s records', + copied_records, + skipped_records, + removed_records) - logging.info('Post recover blob processing: check_result="%s", move_back="%s"', - check_result, - move_back) + logging.info('Post recover blob processing: check_result="%s", move_back="%s"', + check_result, move_back) - if check_result and not BlobRepairer(blob_path).check(verify_csum=True, fast=False): - logging.error("check of fixed blob failed: path='%s'", blob_path) - raise RuntimeError('restored blob check has been failed') + if check_result: + if not BlobRepairer(destination_blob_path).check(verify_csum=True, fast=False): + logging.error("check of fixed blob failed: path='%s'", destination_blob_path) + raise RuntimeError('restored blob check has been failed') - if move_back: - self._blob.close() - self._blob.move_back(from_index=destination_index_path, - from_data=destination_data_path, - to_index=self._blob.index.path, - to_data=self._blob.data.path) + if move_back: + self.move_back(source_blob_path=source_blob_path, + destination_blob=destination_blob) def copy_valid_records(self, destination, overwrite=False, move_back=False, check_result=False): """Recover blob by copying only valid records from blob.""" - basename = os.path.basename(self._blob.data.path) - blob_path = os.path.join(destination, basename) - if not is_destination_writable(self._blob.data.path, blob_path, overwrite): - raise RuntimeError("can't copy valid records to already existing blob file: {}".format(blob_path)) + source_blob_path = self._blob.data.path + + basename = os.path.basename(source_blob_path) + destination_blob_path = os.path.join(destination, basename) + + if not is_destination_writable(source_blob_path, destination_blob_path, overwrite): + raise RuntimeError("can't copy valid records to already existing blob file: {}".format(destination_blob_path)) self._index_headers += self._stat.data_recoverable_headers - logging.info('Recovering blob %s -> %s', self._blob.data.path, blob_path) + logging.info('Recovering blob %s -> %s', source_blob_path, destination_blob_path) copied_records = 0 copied_size = 0 - with managed_blob(blob_path) as dst_blob: - - destination_index_path, destination_blob_path = dst_blob.get_index_data_path_tuple() + with managed_blob(destination_blob_path, self._sorted_index_headers) as destination_blob: for header in self._index_headers: - copy_record(self._blob, dst_blob, header) + copy_record(self._blob, destination_blob, header) copied_records += 1 copied_size += header.disk_size logging.info('I have copied %s (%s) records %s -> %s ', copied_records, sizeof_fmt(copied_size), - self._blob.data.path, - blob_path) + source_blob_path, + destination_blob_path) - if check_result and not BlobRepairer(blob_path).check(verify_csum=True, fast=False): - logging.error("check of fixed blob failed: path='%s'", blob_path) - raise RuntimeError('restored blob check has been failed') + if check_result: + if not BlobRepairer(destination_blob_path).check(verify_csum=True, fast=False): + logging.error("Check of fixed blob failed: path='%s'", destination_blob_path) + raise RuntimeError('restored blob check has been failed') - if move_back: - # Both destination and source blobs should be closed at this point. - self._blob.close() - to_index, to_data = self._blob.get_index_data_path_tuple() + if move_back: + self._move_back(source_blob_path=source_blob_path, + destination_blob=destination_blob) - self._blob.move_back(from_index=destination_index_path, - from_data=destination_blob_path, - to_index=to_index, - to_data=to_data) + def _move_back(self, source_blob_path, destination_blob): + source_folder = os.path.abspath(source_blob_path) + source_folder = os.path.dirname(source_folder) + + logging.info('Replacing blob: from="%s", remote_folder="%s"', + destination_blob.data.path, source_folder) + + # NOTE: blob's underlaying files would be deleted permanently. + self._blob.remove() + destination_blob.move_to(source_folder) def fix(self, destination, noprompt, overwrite=False, move_back=False, check_result=False): """Check blob's data & index and try to fix them if they are broken. @@ -1309,7 +1429,8 @@ def fix(self, destination, noprompt, overwrite=False, move_back=False, check_res self.recover_index(self._blob.data, destination, overwrite=overwrite, - move_back=move_back) + move_back=move_back, + check_result=check_result) elif click.confirm('Should I try to recover both index and data from {}?' .format(self._blob.data.path), default=True): @@ -1320,9 +1441,9 @@ def fix(self, destination, noprompt, overwrite=False, move_back=False, check_res else: if not self._index_headers: logging.error('Nothing can be recovered from %s, so it should be removed', self._blob.data.path) - filename = '{}{}'.format( - os.path.join(destination, os.path.basename(self._blob.data.path)), - TO_REMOVE_SUFFIX) + filename = os.path.basename(self._blob.data.path) + TO_REMOVE_SUFFIX + filename = os.path.join(destination, filename) + with open(filename, 'wb'): pass elif noprompt or click.confirm('Should I repair {}?'.format(self._blob.data.path), diff --git a/tests/test_blob_repairer.py b/tests/test_blob_repairer.py index e06e47b..62eddda 100644 --- a/tests/test_blob_repairer.py +++ b/tests/test_blob_repairer.py @@ -429,6 +429,8 @@ def test_fix_destination_writable(mocked_blob, mocked_blob.create.return_value = mocked_blob mocked_blob.get_index_data_path_tuple.return_value = (None, None) + type(mocked_blob.return_value.data).path = mock.PropertyMock(return_value='data') + blob_repairer = BlobRepairer('.') if callee == BlobRepairer.recover_index: @@ -445,7 +447,7 @@ def test_fix_destination_writable(mocked_blob, @mock.patch('eblob_kit.is_destination_writable', return_value=False) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) @mock.patch('eblob_kit.Blob', autospec=True) -def test_fix_destination_not_writable(_mocked_blob, +def test_fix_destination_not_writable(mocked_blob, _mocked_open, _mocked_is_writable, callee): @@ -454,6 +456,8 @@ def test_fix_destination_not_writable(_mocked_blob, Checks for `copy_valid_records`, `recover_index` and `recover_blob`. """ + type(mocked_blob.return_value.data).path = mock.PropertyMock(return_value='data') + blob_repairer = BlobRepairer('.') with pytest.raises(RuntimeError): From b985d9f07a95bf1fb75984d4611a75a0871f09be Mon Sep 17 00:00:00 2001 From: karitra Date: Wed, 3 Jul 2019 18:47:52 +0300 Subject: [PATCH 5/6] test: add is_sorted to tests --- tests/test_blob_repairer.py | 7 +------ tests/test_is_sortable.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 tests/test_is_sortable.py diff --git a/tests/test_blob_repairer.py b/tests/test_blob_repairer.py index 62eddda..40a134e 100644 --- a/tests/test_blob_repairer.py +++ b/tests/test_blob_repairer.py @@ -417,7 +417,7 @@ def test_check_index_non_valid(_mocked_exists, @mock.patch('eblob_kit.is_destination_writable', return_value=True) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) @mock.patch('eblob_kit.Blob', autospec=True) -def test_fix_destination_writable(mocked_blob, +def test_fix_destination_writable(_mocked_blob, _mocked_open, _mocked_is_writable, callee): @@ -426,11 +426,6 @@ def test_fix_destination_writable(mocked_blob, Checks for `copy_valid_records`, `recover_index` and `recover_blob`. """ - mocked_blob.create.return_value = mocked_blob - mocked_blob.get_index_data_path_tuple.return_value = (None, None) - - type(mocked_blob.return_value.data).path = mock.PropertyMock(return_value='data') - blob_repairer = BlobRepairer('.') if callee == BlobRepairer.recover_index: diff --git a/tests/test_is_sortable.py b/tests/test_is_sortable.py new file mode 100644 index 0000000..7e76739 --- /dev/null +++ b/tests/test_is_sortable.py @@ -0,0 +1,30 @@ +import eblob_kit + +from collections import namedtuple + + +DummyRecord = namedtuple('DummyRecord', 'a b') + + +def test_sequence_is_sorted_default_key(): + """Test that sequence is sorted with default key selector.""" + sorted_sequence = [-1, 1, 2, 3, 4, 5, 10, 100, 1000] + assert eblob_kit.is_sorted(sorted_sequence) + + +def test_sequence_not_sorted_default_key(): + """Test that sequence is not sorted with default key selector.""" + non_sorted_sequence = [-1, 1, 2, 3, 4, 5, 10, 100, 99, 1000] + assert not eblob_kit.is_sorted(non_sorted_sequence) + + +def test_sequence_is_sorted_custom_key(): + """Test sequence is sorted with custom field selector.""" + sequence_length = 5 + sequence = [ + DummyRecord(i, sequence_length - i) for i in xrange(sequence_length) + ] + + assert eblob_kit.is_sorted(sequence) + assert eblob_kit.is_sorted(sequence, lambda x: x.a) + assert not eblob_kit.is_sorted(sequence, lambda x: x.b) From 81e86a906427a35b5a7d7be0500b237d65cfd31c Mon Sep 17 00:00:00 2001 From: karitra Date: Tue, 2 Jul 2019 02:38:04 +0300 Subject: [PATCH 6/6] v0.1.7 --- debian/changelog | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 0d6f293..6c8c022 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +eblob-kit (0.1.7) unstable; urgency=medium + + * feat: check_result flag to verify fix results + * feat: option to move restored data to origins + * test: move_back option support + + -- Alex Karev Tue, 02 Jul 2019 02:32:57 +0300 + eblob-kit (0.1.6) unstable; urgency=medium * fix: make output of `fix_blob` and `fix` command similar diff --git a/setup.py b/setup.py index 4e46013..bd72e6a 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def run_tests(self): setup(name='eblob_kit', - version='0.1.6', + version='0.1.7', author='Kirill Smorodinnikov', author_email='shaitkir@gmail.com', py_modules=['eblob_kit'],