diff --git a/README.md b/README.md index 09835c8..875473e 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,9 @@ Release candidate 1 major enhancements: - Btrfs and XFS reflink support - Authenticated encryption with auth caching - + - Full data & metadata integrity checking - + - Fast differential receive based on available snapshots - Overall faster operation @@ -356,6 +356,7 @@ the volume data if present. --sparse | Receive volume data sparsely (implies --sparse-write) --sparse-write | Overwrite local data only where it differs (receive) --use-snapshot | Use snapshots when available for faster `receive`. +--send-unchanged | Record unchanged volumes, don't skip them (send) --unattended, -u | Don't prompt for interactive input. --clean | Perform garbage collection (arch-check) or metadata removal (delete). --verbose | Increase details. @@ -369,10 +370,12 @@ the volume data if present. --save-to=_path_ | Save volume to _path_ (receive). --local_from=_json file_ | Specify local:[volumes] sets instead of --local. --import-other-from | Import volume data from a non-snapshot capable path during `send` ---encrypt=_cipher_ | Set encryption mode or _'off'_ (default: _'xchacha20-t3'_) +--session-strict=_on|off_ | Don't retrieve volume from next-oldest session if no exact session match +--encrypt=_cipher_ | Set encryption mode or _'off'_ (default: _'xchacha20-dgr'_) --compression | (arch-init) Set compression type:level. --hashtype | (arch-init) Set data hash algorithm: _hmac-sha256_ or _blake2b_. --chunk-factor | (arch-init) Set archive chunk size. +--vid | Select volume by ID (delete) --tar-bypass | Use direct access for file:/ archives (send) --passcmd=_'command'_ | Read passphrase from output of a wallet/auth app --upgrade-format | Upgrade older Wyng archive to current format. (arch-check) @@ -551,6 +554,13 @@ which means this option cannot be used to `send` directly to volume names in the contain that character sequence. +`--session-strict=on|off` + +For receive, verify, diff: If set to 'on' (the default) Wyng won't retrieve volumes from next-oldest session if the +specified volumes don't have an exact match for the specified session. When set to 'off' +Wyng will try to retrieve the next-oldest version of the volume if one exists. + + `--local_from=_json file_` Specify both local storage and volume names for `send` or `receive` as sets, instead @@ -785,6 +795,9 @@ root files in each 'a_*' directory are part of Wyng's defense against rollback a feel the need to manually reclaim space used in this dir then consider leaving the _archive.\*_ files in place. +* If data corruption in the archive is suspected, use `wyng arch-check` which will scan for errors and present you with options for recovery. + +* If a volume becomes damaged and unrecoverable it may be necessary to delete it by its volume ID by using `wyng delete --vid` instead of the volume name. ### Testing diff --git a/src/wyng b/src/wyng index d51c39d..cb56418 100755 --- a/src/wyng +++ b/src/wyng @@ -222,7 +222,7 @@ class ArchiveSet: # load volume objects fully self.load_volumes(children) - def load_volumes(self, children, vnames=[], vids=[], handle=False): + def load_volumes(self, children, vids=[], handle=False): errors = [] ; vids = vids or list(self.conf["volumes"].keys()) recv_list = [(x+"/volinfo", ArchiveVolume.max_volinfosz) @@ -330,10 +330,13 @@ class ArchiveSet: vol.save_volinfo(ext) return vol - def delete_volume_meta(self, datavol): + def delete_volume_meta(self, vname=None, vid=None): # Enh: add delete-by-vid - vid = self.vols[datavol].vid ; vpath = self.vols[datavol].path - del(self.vols[datavol], self.conf["volumes"][vid]) + vid = self.vols[vname].vid if vname else vid + assert vid.startswith("Vol_") + vpath = self.path+"/"+vid + del(self.conf["volumes"][vid]) + if vname in self.vols: del(self.vols[datavol]) self.save_conf() if exists(vpath): shutil.rmtree(vpath) @@ -1959,6 +1962,7 @@ def arch_check(storage, aset, vol_list=None, startup=False, upgrade=False): for v, e in v_errs: err_out(v+" - "+repr(e)); error_cache.append(v) + # Enh: Offer to reconstruct volinfo from available session data if v_errs: err_out(f"\n{vid} cannot be recovered.") continue @@ -1968,19 +1972,24 @@ def arch_check(storage, aset, vol_list=None, startup=False, upgrade=False): ses_errs = vol.load_sessions(handle=True, force=True, mfdecode=True) seslist = list(vol.sessions.values()) ; removed = [] + # un-loaded sessions cannot be sorted, so remove them from seslist for ses, sn, e in ((vol.sessions[x], x, y) for x, y in ses_errs): if ses in seslist: del(seslist[seslist.index(ses)]) removed.append(sn) ; err_out(vol.name+" - "+sn+" - "+repr(e)) if ses_errs: - error_cache.append(vol.name) ; seslist.sort(key=lambda x: x.sequence) + # find last good session and reset volume to it: + # flag volume and sort the session list by sequence# + error_cache.append(vol.name) ; seslist.sort(key=lambda x: x.sequence) ; si = -1 + + # find longest coherent chain of sessions for si in range(len(seslist)): if not check_session_seq(vol, seslist=seslist, seq=si): break else: si += 1 - if si == 0: + if si < 1: err_out(f"\nVolume '{vol.name}' ({vid}) cannot be recovered.") else: sgood = seslist[si-1].name @@ -2111,6 +2120,9 @@ def remove_local_snapshots(storage, archive, del_all=False, purge=None): purge_other = purge == "other" ; del_all = purge == "full" assert not (purge_other and not archive) + if not storage.arch_vols: + storage.update_vol_list({x.name: x.vid for x in archive.vols.values()}) + # Remove LVM snapshots for lv in (x for vg in storage.vgs_all.values() for x in vg.values()): if lv.name.endswith(lv.snap_ext) and "wyng" in lv.tags.split(","): @@ -2258,9 +2270,9 @@ def get_configs_remote(dest, base_dir, opts): shutil.copyfile(pjoin(tmpdir,f), pjoin(arch_dir,f)) # Initial auth successful! Fetch + auth volume metadata... - if opts.action in ("arch-check"): + if opts.action in ("arch-check","delete"): depth = 0 - elif opts.action in ("delete","add"): + elif opts.action in ("add",): depth = 1 else: depth = 2 @@ -2486,7 +2498,7 @@ class Destination: import os, sys, time, signal, shutil, subprocess as SPr, gzip, tarfile def fssync(path, force=False): - if msync or force: SPr.Popen(["sync","-f",path]) + if msync or force: misc_procs.append(SPr.Popen(["sync","-f",path])) def catch_signals(sel=["INT","TERM","QUIT","ABRT","ALRM","TSTP","USR1"], iflag=False): for sval in (getattr(signal,"SIG"+x) for x in sel): @@ -2605,7 +2617,7 @@ def helper_merge_finalize(): ## MAIN ## cmd = sys.argv[1] ; msync = "--sync" in sys.argv ; magic = b"\xff\x11\x15" -tmpdir = os.path.dirname(os.path.abspath(sys.argv[0])) +tmpdir = os.path.dirname(os.path.abspath(sys.argv[0])) ; misc_procs = [] exists = os.path.exists ; replace = os.replace ; remove = os.remove if cmd in ("receive","dedup") and exists(tmpdir+"/dest.lst.gz"): lstf = gzip.open(tmpdir+"/dest.lst.gz", "rt") @@ -4620,43 +4632,50 @@ def add_volume(aset, datavol, desc): # Remove a volume from the archive -def delete_volume(storage, aset, dv, vid=None): +def delete_volume(storage, aset, dv=None, vid=None): + assert not (dv and vid) ; inproc = aset.in_process and aset.in_process[0] == "delete" if not storage.online: print("Note: A valid --local location was not specified; no volume snapshots" " will be removed.\n") if vid: - vols = [x for x in aset.vols.values() if x.vid == vid] - if not vols: err_out(f"Volume {vid} not found.") ; return - dv = vols[0].name + if vid not in aset.conf["volumes"] or not vid.startswith("Vol_"): + if inproc: aset.set_in_process(None) + err_out(f"Vid '{vid}' not found.") + else: + aset.load_volumes(1) + if dv not in aset.vols: + x_it(1, f"Volume '{dv}' not found.") + vid = aset.vols[dv].vid - if not options.unattended and not options.force and not aset.in_process: + if not options.unattended and not options.force and not inproc: print("\nWarning! Delete will remove ALL metadata AND archived data", - "for volume", dv) + f"for volume '{dv or vid}'") ans = ask_input("Are you sure? [y/N]: ") if ans.lower() not in {"y","yes"}: x_it(0,"") - print() - if not (vol := aset.vols.get(dv)): - err_out(f"Volume {dv} not found.") ; return - - print(f"Deleting volume '{dv}' ({vol.vid}) from archive.") - catch_signals() - if not aset.in_process: aset.set_in_process(["delete", vol.vid]) - dvid = aset.delete_volume_meta(dv) - update_dest(aset, pathlist=[aset.confname]) - aset.set_in_process(None) + if vid in aset.conf["volumes"]: + print(f"\nDeleting volume '{dv or vid}' from archive.") + catch_signals() ; inproc = True + aset.set_in_process(["delete", vid]) + dvid = aset.delete_volume_meta(vid=vid) + update_dest(aset, pathlist=[aset.confname]) + aset.set_in_process(None) + catch_signals(**signormal) - if dvid: + if dvid.startswith("Vol_") and inproc: aset.dest.run(["rm -rf '%s'" % dvid], destcd=aset.dest.path) - catch_signals(**signormal) - if storage.online: + if storage.online and dv: + storage.new_vol_entry(dv, dvid) for lvol_name in (storage.lvols[dv].snap1, storage.lvols[dv].snap2): storage.lvols[lvol_name].delete() - return + else: + print("(Skipping snapshot removal.)") + + x_it(int(not inproc)) def show_list(aset, selected_vols): @@ -4840,7 +4859,7 @@ def cleanup(): # Constants / Globals prog_name = "wyng" -prog_version = "0.8 beta" ; prog_date = "20240607" +prog_version = "0.8 beta" ; prog_date = "20240608" format_version = 3 ; debug = False admin_permission = os.getuid() == 0 @@ -4922,6 +4941,7 @@ parser_defs = [ [["--hashtype"], {"default": "", "help": "Init: hash function type"}], [["--chunk-factor"], {"dest": "chfactor", "help": "Init: set chunk size to N*64kB"}], + [["--vid"], {"help": "Reference a volume by ID number (delete)"}], [["--passcmd"], {"help": "Command to fetch auth passphrase"}], [["--meta-dir"], {"dest": "metadir", "default": "", "help": "Use alternate metadata path"}], [["--meta-reduce"], {"default": "on:3000", "help": "Metadata retention policy (see doc)"}], @@ -5033,7 +5053,7 @@ if aset.in_process and dest.online \ print("Completing prior operation in progress:", " ".join(aset.in_process[0:2])) if aset.in_process[0] == "delete": - delete_volume(storage, aset, "", vid=aset.in_process[1]) + delete_volume(storage, aset, vid=aset.in_process[1]) elif aset.in_process[0] == "merge": vname = [x.name for x in aset.vols.values() if x.vid == aset.in_process[1]][0] @@ -5059,10 +5079,10 @@ print("Encrypted" if (aset.mcrypto and aset.datacrypto) else "Un-encrypted", exclude_vols = set(options.volex or []) datavols = sorted(set(aset.vols.keys()) - exclude_vols) if options.all else [] selected_vols = sorted(set(options.volumes[:] + datavols)) -for vol in selected_vols[:]: - if vol not in aset.vols and options.action not in {"add","rename","send"}: - print(f"Volume '{vol}' not found; Skipping.") - selected_vols.remove(vol) +for v in selected_vols[:]: + if v not in aset.vols and options.action not in {"add","delete","rename","send","arch-check"}: + print(f"Volume '{v}' not found; Skipping.") + selected_vols.remove(v) ## Process Commands ## @@ -5176,14 +5196,12 @@ elif options.action == "delete": if options.all: meta_reduce, meta_min = "extra", 0 remove_local_snapshots(storage, None if options.all else aset, purge=options.purge) - elif len(options.volumes) != 1: + elif len(options.volumes) + bool(options.vid) != 1: x_it(1, "Specify one volume to delete.") - elif options.volumes[0] not in aset.vols: - x_it(1, "Volume not found.") - else: - delete_volume(storage, aset, options.volumes[0]) + delete_volume(storage, aset, **{"vid": options.vid} if options.vid + else { "dv": options.volumes[0]}) elif options.action == "arch-init": diff --git a/src/wyng.gpg b/src/wyng.gpg index b6c549f..0754b40 100644 Binary files a/src/wyng.gpg and b/src/wyng.gpg differ