From d6aaf8ea6918dbfb5e3434e7cc35681bb2f35159 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Fri, 24 Jul 2020 23:15:15 +0200 Subject: [PATCH 01/12] new release cycle --- CHANGES.md | 4 ++++ src/abgleich/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 550d143..fa0630b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Changes +## 0.0.7 (2020-XX-XX) + +- (TBD) + ## 0.0.6 (2020-07-24) - FIX: Development installs with `make install` do no longer fail, see #17. diff --git a/src/abgleich/__init__.py b/src/abgleich/__init__.py index a17d78c..adae71e 100644 --- a/src/abgleich/__init__.py +++ b/src/abgleich/__init__.py @@ -28,4 +28,4 @@ # META # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -__version__ = "0.0.6" +__version__ = "0.0.7" From def351d07bcbb8329a07ffef0344bced06336ba8 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Fri, 24 Jul 2020 23:17:12 +0200 Subject: [PATCH 02/12] black --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0c5d394..c56a060 100644 --- a/setup.py +++ b/setup.py @@ -63,10 +63,12 @@ def get_version(code): continue if len(item.targets) != 1: continue - if item.targets[0].id != '__version__': + if item.targets[0].id != "__version__": continue return item.value.s -with open(os.path.join(SRC_DIR, 'abgleich', '__init__.py'), 'r', encoding = 'utf-8') as f: + + +with open(os.path.join(SRC_DIR, "abgleich", "__init__.py"), "r", encoding="utf-8") as f: __version__ = get_version(f.read()) # Requirements From 80229944473381cbd309bc788825f42197c62199 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Fri, 24 Jul 2020 23:20:15 +0200 Subject: [PATCH 03/12] updated documentation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 19dc342..c8f1a1c 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ target: prefix: BACKUP_SOMEMACHINE host: bigdata user: zfsadmin +include_root: yes keep_snapshots: 2 always_changed: no written_threshold: 1048576 @@ -78,7 +79,7 @@ ssh: cipher: aes256-gcm@openssh.com ``` -The prefix can be empty on either side. If a `host` is set to `localhost`, the `user` field can be left empty. Both source and target can be remote hosts or localhost at the same time. `keep_snapshots` is an integer and must be greater or equal to `1`. It specifies the number of snapshots that are kept per dataset on the source side when a cleanup operation is triggered. `suffix` contains the name suffix for new snapshots. Setting `always_changed` to `yes` causes `abgleich` to beliefe that all datasets have always changed since the last snapshot, completely ignoring what ZFS actually reports. No diff will be produced & checked for values of `written` lower than `written_threshold`. Checking diffs can be completely deactivated by setting `check_diff` to `no`. `digits` specifies how many digits are used for a decimal number describing the n-th snapshot per dataset per day as part of the name of new snapshots. `ignore` lists stuff underneath the `prefix` which will be ignored by this tool, i.e. no snapshots, backups or cleanups. `ssh` allows to fine-tune the speed of backups. In fast local networks, it is best to set `compression` to `no` because the compression is usually slowing down the transfer. However, for low-bandwidth transmissions, it makes sense to set it to `yes`. For significantly better speed in fast local networks, make sure that both the source and the target system support a common cipher, which is accelerated by [AES-NI](https://en.wikipedia.org/wiki/AES_instruction_set) on both ends. +The prefix can be empty on either side. If a `host` is set to `localhost`, the `user` field can be left empty. Both source and target can be remote hosts or localhost at the same time. `include_root` indicates whether `{zpool}{/{prefix}}` should be included in all operations. `keep_snapshots` is an integer and must be greater or equal to `1`. It specifies the number of snapshots that are kept per dataset on the source side when a cleanup operation is triggered. `suffix` contains the name suffix for new snapshots. Setting `always_changed` to `yes` causes `abgleich` to beliefe that all datasets have always changed since the last snapshot, completely ignoring what ZFS actually reports. No diff will be produced & checked for values of `written` lower than `written_threshold`. Checking diffs can be completely deactivated by setting `check_diff` to `no`. `digits` specifies how many digits are used for a decimal number describing the n-th snapshot per dataset per day as part of the name of new snapshots. `ignore` lists stuff underneath the `prefix` which will be ignored by this tool, i.e. no snapshots, backups or cleanups. `ssh` allows to fine-tune the speed of backups. In fast local networks, it is best to set `compression` to `no` because the compression is usually slowing down the transfer. However, for low-bandwidth transmissions, it makes sense to set it to `yes`. For significantly better speed in fast local networks, make sure that both the source and the target system support a common cipher, which is accelerated by [AES-NI](https://en.wikipedia.org/wiki/AES_instruction_set) on both ends. ## USAGE From 7514a8f43f191ea3a0c5e665011b86a0319c9f6c Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Tue, 4 Aug 2020 20:04:18 +0200 Subject: [PATCH 04/12] fix #18 allow empty tables --- src/abgleich/core/zpool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/abgleich/core/zpool.py b/src/abgleich/core/zpool.py index 9c4b596..730d2e8 100644 --- a/src/abgleich/core/zpool.py +++ b/src/abgleich/core/zpool.py @@ -272,6 +272,10 @@ def print_table(self): for snapshot in dataset.snapshots: table.append(self._table_row(snapshot)) + if len(table) == 0: + print('(empty)') + return + print( tabulate( table, From 4b7c9dfb45baaf7c2554b3a838fefc1f90a4ac2f Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Tue, 4 Aug 2020 20:15:57 +0200 Subject: [PATCH 05/12] attempt to fix #19 allow empty datasets --- src/abgleich/core/zpool.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/abgleich/core/zpool.py b/src/abgleich/core/zpool.py index 730d2e8..7f5bf70 100644 --- a/src/abgleich/core/zpool.py +++ b/src/abgleich/core/zpool.py @@ -367,9 +367,15 @@ def from_config(cls, side: str, config: ConfigABC,) -> ZpoolABC: root_dataset = root(config[side]["zpool"], config[side]["prefix"]) - output, _ = Command.on_side( - ["zfs", "get", "all", "-r", "-H", "-p", root_dataset,], side, config, - ).run() + try: + output, error = Command.on_side( + ["zfs", "get", "all", "-r", "-H", "-p", root_dataset,], side, config, + ).run() + except SystemError as e: + if 'dataset does not exist' not in error: + raise e + return cls(datasets=[], side=side, config=config,) + output = [ line.split("\t") for line in output.split("\n") if len(line.strip()) > 0 ] From f385a9f4fa2ba8cacdece712261974ea884fd44f Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Tue, 4 Aug 2020 20:19:21 +0200 Subject: [PATCH 06/12] attempt to fix #19 allow empty datasets (2) --- src/abgleich/core/zpool.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/abgleich/core/zpool.py b/src/abgleich/core/zpool.py index 7f5bf70..68eff7e 100644 --- a/src/abgleich/core/zpool.py +++ b/src/abgleich/core/zpool.py @@ -368,13 +368,14 @@ def from_config(cls, side: str, config: ConfigABC,) -> ZpoolABC: root_dataset = root(config[side]["zpool"], config[side]["prefix"]) try: - output, error = Command.on_side( + output, _ = Command.on_side( ["zfs", "get", "all", "-r", "-H", "-p", root_dataset,], side, config, ).run() except SystemError as e: - if 'dataset does not exist' not in error: - raise e - return cls(datasets=[], side=side, config=config,) + if len(e.args) >= 4: + if 'dataset does not exist' in e.args[3]: + return cls(datasets=[], side=side, config=config,) + raise e output = [ line.split("\t") for line in output.split("\n") if len(line.strip()) > 0 From 25537c634a70e117d0e1a8a2e1e34a292cfdf501 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 11:46:44 +0200 Subject: [PATCH 07/12] fix #20 host-up check also for target if required --- src/abgleich/cli/tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/abgleich/cli/tree.py b/src/abgleich/cli/tree.py index 8558d73..7348b59 100644 --- a/src/abgleich/cli/tree.py +++ b/src/abgleich/cli/tree.py @@ -49,8 +49,8 @@ def tree(configfile, side): config = Config.from_fd(configfile) - if not is_host_up("source", config): - print(f'{t("host is not up"):s}: source') + if not is_host_up(side, config): + print(f'{t("host is not up"):s}: {side:s}') sys.exit(1) zpool = Zpool.from_config(side, config=config) From 350de43cae7def8945511dfc1340f6646d782615 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 11:51:07 +0200 Subject: [PATCH 08/12] command exposes exception if asked for return code --- src/abgleich/core/command.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/abgleich/core/command.py b/src/abgleich/core/command.py index bdbcbea..d3c3ab0 100644 --- a/src/abgleich/core/command.py +++ b/src/abgleich/core/command.py @@ -52,7 +52,7 @@ def __str__(self) -> str: def run( self, returncode: bool = False - ) -> typing.Union[typing.Tuple[str, str], typing.Tuple[str, str, int]]: + ) -> typing.Union[typing.Tuple[str, str], typing.Tuple[str, str, int, Exception]]: proc = subprocess.Popen( self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -61,11 +61,13 @@ def run( status = not bool(proc.returncode) output, errors = output.decode("utf-8"), errors.decode("utf-8") + exception = SystemError("command failed", str(self), output, errors) + if returncode: - return output, errors, int(proc.returncode) + return output, errors, int(proc.returncode), exception if not status or len(errors.strip()) > 0: - raise SystemError("command failed", str(self), output, errors) + raise exception return output, errors From ba20ac96a83d79848b891467f1e181fda3e6b6ca Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 11:51:35 +0200 Subject: [PATCH 09/12] new command run return value added --- src/abgleich/core/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abgleich/core/lib.py b/src/abgleich/core/lib.py index 76632a5..3f7b4ad 100644 --- a/src/abgleich/core/lib.py +++ b/src/abgleich/core/lib.py @@ -48,7 +48,7 @@ def is_host_up(side: str, config: ConfigABC) -> bool: if config[side]["host"] == "localhost": return True - _, _, returncode = Command.on_side(["exit"], side, config).run(returncode=True) + _, _, returncode, _ = Command.on_side(["exit"], side, config).run(returncode=True) assert returncode in (0, 255) return returncode == 0 From ab93fe323f9c597f049048bef72aadfa9b349d1b Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 11:53:42 +0200 Subject: [PATCH 10/12] continue to catch empty datasets but raise exception for all other errors --- src/abgleich/core/zpool.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/abgleich/core/zpool.py b/src/abgleich/core/zpool.py index 68eff7e..5cefc47 100644 --- a/src/abgleich/core/zpool.py +++ b/src/abgleich/core/zpool.py @@ -367,15 +367,14 @@ def from_config(cls, side: str, config: ConfigABC,) -> ZpoolABC: root_dataset = root(config[side]["zpool"], config[side]["prefix"]) - try: - output, _ = Command.on_side( - ["zfs", "get", "all", "-r", "-H", "-p", root_dataset,], side, config, - ).run() - except SystemError as e: - if len(e.args) >= 4: - if 'dataset does not exist' in e.args[3]: - return cls(datasets=[], side=side, config=config,) - raise e + output, errors, returncode, exception = Command.on_side( + ["zfs", "get", "all", "-r", "-H", "-p", root_dataset,], side, config, + ).run(returncode = True) + + if returncode != 0 and 'dataset does not exist' in errors: + return cls(datasets=[], side=side, config=config,) + if returncode != 0: + raise exception output = [ line.split("\t") for line in output.split("\n") if len(line.strip()) > 0 From ef5ac16b989e1f77653d1634711522aff7ee8e60 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 11:53:52 +0200 Subject: [PATCH 11/12] log changes --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index fa0630b..46dd41d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,9 @@ ## 0.0.7 (2020-XX-XX) -- (TBD) +- FIX: `tree` now property checks if source or target is up, depending on what a user wants to see, see #20. +- FIX: All `abgleich` commands can properly initialize (instead of crashing) if the target tree is empty, see #19. +- FIX: `tree` shows message if there is no tree instead of crashing, see #18. ## 0.0.6 (2020-07-24) From e49502e4d5422c17d2a09d750c268d6a0b60dab8 Mon Sep 17 00:00:00 2001 From: "Sebastian M. Ernst" Date: Wed, 5 Aug 2020 12:02:24 +0200 Subject: [PATCH 12/12] prepare for release --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 46dd41d..1ca06e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Changes -## 0.0.7 (2020-XX-XX) +## 0.0.7 (2020-08-05) - FIX: `tree` now property checks if source or target is up, depending on what a user wants to see, see #20. - FIX: All `abgleich` commands can properly initialize (instead of crashing) if the target tree is empty, see #19.