From 551525d846e4a023427f189d599c0e88e82f9c91 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 11 Feb 2023 18:59:17 -0800 Subject: [PATCH 01/86] Abstracted UI, errors through tracing, metrics, and metric and error json (#197) Add `--trace-time`, `--log-json`, and `--metric-json`. Use `tracing` macros to report errors rather than UI-specific things. Separate out everything connected to terminal/nutmeg progress and output. Return 2 if non-fatal errors or warnings occurred. Respect `$CLICOLORS`. --- Cargo.lock | 281 +++++++++++++++++++++++++++++++++- Cargo.toml | 16 +- src/archive.rs | 197 ++++++++++++------------ src/backup.rs | 88 +++++------ src/band.rs | 27 ++-- src/bandid.rs | 4 +- src/bin/conserve.rs | 125 +++++++++------ src/blockdir.rs | 138 ++++++++--------- src/errors.rs | 168 +++++++++++++++++--- src/index.rs | 23 ++- src/lib.rs | 12 +- src/live_tree.rs | 41 ++--- src/metric_recorder.rs | 152 ++++++++++++++++++ src/misc.rs | 83 +++++++++- src/progress.rs | 153 ++++++++++++++++++ src/progress/term.rs | 207 +++++++++++++++++++++++++ src/restore.rs | 87 ++++------- src/show.rs | 12 +- src/stats.rs | 69 +-------- src/trace_counter.rs | 41 +++++ src/transport.rs | 3 + src/transport/local.rs | 4 + src/tree.rs | 37 ++--- src/ui.rs | 163 +------------------- src/ui/termui.rs | 95 ++++++++++++ src/unix_mode.rs | 18 +-- src/validate.rs | 111 +++++++------- tests/api/backup.rs | 8 +- tests/api/damaged.rs | 22 +-- tests/api/old_archives.rs | 9 +- tests/cli/delete.rs | 17 +- tests/cli/main.rs | 37 ++--- tests/cli/trace.rs | 21 +++ tests/cli/unix/permissions.rs | 18 +-- tests/cli/validate.rs | 64 +++++++- 35 files changed, 1755 insertions(+), 796 deletions(-) create mode 100644 src/metric_recorder.rs create mode 100644 src/progress.rs create mode 100644 src/progress/term.rs create mode 100644 src/trace_counter.rs create mode 100644 src/ui/termui.rs create mode 100644 tests/cli/trace.rs diff --git a/Cargo.lock b/Cargo.lock index 13aacafb..684a0537 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -48,6 +59,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "atty" version = "0.2.14" @@ -108,6 +125,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "byteorder" version = "1.4.3" @@ -179,16 +202,30 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clicolors-control" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e" +dependencies = [ + "atty", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "conserve" version = "23.1.1" dependencies = [ "assert_cmd", "assert_fs", + "assert_matches", "blake2-rfc", "bytes", "cachedir", "clap", + "clicolors-control", "cp_r", "derive_more", "dir-assert", @@ -198,6 +235,8 @@ dependencies = [ "indoc", "itertools", "lazy_static", + "metrics", + "metrics-util", "mutants", "nix", "nutmeg", @@ -342,6 +381,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "errno" version = "0.2.8" @@ -416,7 +461,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -443,6 +488,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + [[package]] name = "heck" version = "0.4.0" @@ -500,6 +554,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "indoc" version = "1.0.8" @@ -552,6 +616,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -589,6 +662,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -613,12 +695,64 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" +dependencies = [ + "ahash", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" +dependencies = [ + "proc-macro2 1.0.50", + "quote 1.0.23", + "syn 1.0.107", +] + +[[package]] +name = "metrics-util" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown", + "indexmap", + "metrics", + "num_cpus", + "ordered-float", + "parking_lot", + "portable-atomic", + "quanta", + "radix_trie", + "sketches-ddsketch", +] + [[package]] name = "mutants" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc0287524726960e07b119cebd01678f852f147742ae0d925e6a520dca956126" +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.26.2" @@ -685,9 +819,8 @@ dependencies = [ [[package]] name = "nutmeg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cf1f0653933873dfd8eccc0ac30b6e12d1db895a4d0dd868d81ce4105400ea" +version = "0.1.3-pre" +source = "git+https://github.com/sourcefrog/nutmeg#2d10f07580e038040f9ea72b0cc9cc65b3316104" dependencies = [ "atty", "parking_lot", @@ -701,6 +834,15 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +[[package]] +name = "ordered-float" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +dependencies = [ + "num-traits", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -763,6 +905,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -884,6 +1032,22 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "quanta" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -914,6 +1078,16 @@ dependencies = [ "proc-macro2 1.0.50", ] +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" @@ -953,6 +1127,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "raw-cpuid" +version = "10.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb" +dependencies = [ + "bitflags", +] + [[package]] name = "rayon" version = "1.6.1" @@ -1127,6 +1310,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" + [[package]] name = "smallvec" version = "1.10.0" @@ -1346,6 +1535,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.16" @@ -1356,12 +1555,16 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", + "time", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -1473,12 +1676,82 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2 1.0.50", + "quote 1.0.23", + "syn 1.0.107", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote 1.0.23", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2 1.0.50", + "quote 1.0.23", + "syn 1.0.107", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 16edd0d7..17b417d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,17 +16,20 @@ doc = false name = "conserve" [dependencies] +assert_matches = "1.5.0" blake2-rfc = "0.2.18" bytes = "1.1.0" cachedir = "0.3" +clicolors-control = "1.0" derive_more = "0.99" filetime = "0.2" globset = "0.4.5" hex = "0.4.2" itertools = "0.10" lazy_static = "1.4.0" +metrics = "0.20" +metrics-util = "0.14" mutants = "0.0.3" -nutmeg = "0.1" rayon = "1.3.0" readahead-iterator = "0.1.1" regex = "1.3.9" @@ -40,7 +43,6 @@ thousands = "0.2.0" time = { version = "0.3", features = ["local-offset"] } tracing = "0.1" tracing-appender = "0.2" -tracing-subscriber = { version = "0.3.11", features = ["env-filter", "fmt"] } unix_mode = "0.1" url = "2.2.2" indoc = "1.0.8" @@ -53,6 +55,14 @@ nix = "0.26" version = "4.0" features = ["derive", "deprecated", "wrap_help"] +[dependencies.nutmeg] +version = "0.1.3-pre" +git = "https://github.com/sourcefrog/nutmeg" + +[dependencies.tracing-subscriber] +version = "0.3.16" +features = ["env-filter", "fmt", "json", "local-time", "time"] + [dev-dependencies] assert_cmd = "2.0" assert_fs = "1.0" @@ -62,7 +72,7 @@ predicates = "2" pretty_assertions = "1.0" proptest = "1.0" proptest-derive = "0.3" -tracing-test = "0.2" +tracing-test = { version = "0.2", features = ["no-env-filter"] } [features] blake2_simd_asm = ["blake2-rfc/simd_asm"] diff --git a/src/archive.rs b/src/archive.rs index 7f83a791..7d895798 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -18,18 +18,19 @@ use std::io::ErrorKind; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; use itertools::Itertools; -use nutmeg::models::{LinearModel, UnboundedModel}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use tracing::{debug, error, warn}; use crate::blockhash::BlockHash; use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; -use crate::stats::ValidateStats; +use crate::progress::{Bar, Progress}; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; use crate::*; @@ -195,17 +196,29 @@ impl Archive { /// Shows a progress bar as they're collected. pub fn referenced_blocks(&self, band_ids: &[BandId]) -> Result> { let archive = self.clone(); - let progress = nutmeg::View::new( - LinearModel::new("Find referenced blocks in band", band_ids.len()), - ui::nutmeg_options(), - ); + // TODO: Percentage completion based on how many bands have been checked so far. + let bar = Bar::new(); + let references_found = AtomicUsize::new(0); + let bands_started = AtomicUsize::new(0); + let total_bands = band_ids.len(); + let start = Instant::now(); Ok(band_ids .par_iter() - .inspect(move |_| progress.update(|model| model.increment(1))) + .inspect(|_| { + bands_started.fetch_add(1, Ordering::Relaxed); + }) .map(move |band_id| Band::open(&archive, band_id).expect("Failed to open band")) .flat_map_iter(|band| band.index().iter_entries()) .flat_map_iter(|entry| entry.addrs) .map(|addr| addr.hash) + .inspect(|_hash| { + bar.post(Progress::ReferencedBlocks { + references_found: references_found.fetch_add(1, Ordering::Relaxed), + bands_started: bands_started.load(Ordering::Relaxed), + total_bands, + start, + }) + }) .collect()) } @@ -236,58 +249,66 @@ impl Archive { } else { gc_lock::GarbageCollectionLock::new(self)? }; + debug!("Got gc lock"); let block_dir = self.block_dir(); + debug!("List band ids..."); let mut keep_band_ids = self.list_band_ids()?; keep_band_ids.retain(|b| !delete_band_ids.contains(b)); + debug!("List referenced blocks..."); let referenced = self.referenced_blocks(&keep_band_ids)?; - let progress = nutmeg::View::new( - UnboundedModel::new("Find present blocks"), - ui::nutmeg_options(), - ); - let unref = self - .block_dir() - .block_names()? - .inspect(|_| progress.update(|model| model.increment(1))) - .filter(|bh| !referenced.contains(bh)) - .collect_vec(); - drop(progress); + debug!(referenced.len = referenced.len()); + + debug!("Find present blocks..."); + let present = self.block_dir.block_names_set()?; + debug!(present.len = present.len()); + + debug!("Find unreferenced blocks..."); + let unref = present.difference(&referenced).collect_vec(); let unref_count = unref.len(); + debug!(unref_count); stats.unreferenced_block_count = unref_count; - let progress = nutmeg::View::new( - LinearModel::new("Measure unreferenced blocks", unref.len()), - ui::nutmeg_options(), - ); + debug!("Measure unreferenced blocks..."); + let measure_bar = Bar::new(); let total_bytes = unref .par_iter() - .inspect(|_| progress.update(|model| model.increment(1))) - .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) + .enumerate() + .inspect(|(i, _)| { + measure_bar.post(Progress::MeasureUnreferenced { + blocks_done: *i, + blocks_total: unref_count, + }) + }) + .map(|(_i, block_id)| block_dir.compressed_size(block_id).unwrap_or_default()) .sum(); + drop(measure_bar); stats.unreferenced_block_bytes = total_bytes; if !options.dry_run { delete_guard.check()?; + let bar = Bar::new(); - let progress = nutmeg::View::new( - LinearModel::new("Delete bands", delete_band_ids.len()), - ui::nutmeg_options(), - ); - for band_id in delete_band_ids { + for (bands_done, band_id) in delete_band_ids.iter().enumerate() { Band::delete(self, band_id)?; stats.deleted_band_count += 1; - progress.update(|model| model.increment(1)); + bar.post(Progress::DeleteBands { + bands_done, + total_bands: delete_band_ids.len(), + }); } - let progress = nutmeg::View::new( - LinearModel::new("Delete blocks", unref_count), - ui::nutmeg_options(), - ); let error_count = unref .par_iter() - .inspect(|_| progress.update(|model| model.increment(1))) - .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) + .enumerate() + .inspect(|(blocks_done, _hash)| { + bar.post(Progress::DeleteBlocks { + blocks_done: *blocks_done, + total_blocks: unref_count, + }) + }) + .filter(|(_, block_hash)| block_dir.delete_block(block_hash).is_err()) .count(); stats.deletion_errors += error_count; stats.deleted_block_count += unref_count - error_count; @@ -297,60 +318,59 @@ impl Archive { Ok(stats) } - pub fn validate(&self, options: &ValidateOptions) -> Result { - let start = Instant::now(); - let mut stats = self.validate_archive_dir()?; + /// Walk the archive to check all invariants. + /// + /// If problems are found, they are emitted as `warn` or `error` level + /// tracing messages. This function only returns an error if validation + /// stops due to a fatal error. + pub fn validate(&self, options: &ValidateOptions) -> Result<()> { + self.validate_archive_dir()?; - ui::println("Count indexes..."); + debug!("List bands..."); let band_ids = self.list_band_ids()?; - ui::println(&format!("Checking {} indexes...", band_ids.len())); + debug!("Check {} bands...", band_ids.len()); // 1. Walk all indexes, collecting a list of (block_hash6, min_length) // values referenced by all the indexes. - let (referenced_lens, ref_stats) = validate::validate_bands(self, &band_ids); - stats += ref_stats; + let referenced_lens = validate::validate_bands(self, &band_ids)?; if options.skip_block_hashes { // 3a. Check that all referenced blocks are present, without spending time reading their // content. - ui::println("List present blocks..."); - // TODO: Just validate blockdir structure. + debug!("List blocks..."); + // TODO: Check for unexpected files or directories in the blockdir. let present_blocks: HashSet = self.block_dir.block_names_set()?; - for block_hash in referenced_lens - .keys() - .filter(|&bh| !present_blocks.contains(bh)) - { - ui::problem(&format!("Block {block_hash:?} is missing")); - stats.block_missing_count += 1; + for block_hash in referenced_lens.keys() { + if !present_blocks.contains(block_hash) { + error!(%block_hash, "Referenced block missing"); + } } } else { // 2. Check the hash of all blocks are correct, and remember how long // the uncompressed data is. - ui::println("Check blockdir..."); - let block_lengths: HashMap = self.block_dir.validate(&mut stats)?; + let block_lengths: HashMap = self.block_dir.validate()?; // 3b. Check that all referenced ranges are inside the present data. for (block_hash, referenced_len) in referenced_lens { - if let Some(actual_len) = block_lengths.get(&block_hash) { - if referenced_len > (*actual_len as u64) { - ui::problem(&format!("Block {block_hash:?} is too short",)); - // TODO: A separate counter; this is worse than just being missing - stats.block_missing_count += 1; + if let Some(&actual_len) = block_lengths.get(&block_hash) { + if referenced_len > actual_len as u64 { + error!( + %block_hash, + referenced_len, + actual_len, + "Block is shorter than referenced length" + ); } } else { - ui::problem(&format!("Block {block_hash:?} is missing")); - stats.block_missing_count += 1; + error!(%block_hash, "Referenced block missing"); } } } - - stats.elapsed = start.elapsed(); - Ok(stats) + Ok(()) } - fn validate_archive_dir(&self) -> Result { + fn validate_archive_dir(&self) -> Result<()> { // TODO: More tests for the problems detected here. - let mut stats = ValidateStats::default(); - ui::println("Check archive top-level directory..."); + debug!("Check archive directory..."); let mut seen_bands = HashSet::::new(); for entry_result in self .transport @@ -363,21 +383,14 @@ impl Archive { name, .. }) => { - if name.eq_ignore_ascii_case(BLOCK_DIR) { - } else if let Ok(band_id) = name.parse() { - if !seen_bands.insert(band_id) { - stats.structure_problems += 1; - ui::problem(&format!( - "Duplicated band directory in {:?}: {name:?}", - self.transport, - )); + if let Ok(band_id) = name.parse::() { + if !seen_bands.insert(band_id.clone()) { + // TODO: Test this + error!(%band_id, "Duplicated band directory"); } - } else { - stats.unexpected_files += 1; - ui::problem(&format!( - "Unexpected directory in {:?}: {name:?}", - self.transport, - )); + } else if !name.eq_ignore_ascii_case(BLOCK_DIR) { + // TODO: The whole path not just the filename + warn!(path = name, "Unexpected subdirectory in archive directory"); } } Ok(DirEntry { @@ -389,26 +402,20 @@ impl Archive { && !name.eq_ignore_ascii_case(crate::gc_lock::GC_LOCK) && !name.eq_ignore_ascii_case(".DS_Store") { - stats.unexpected_files += 1; - ui::problem(&format!( - "Unexpected file in archive directory {:?}: {name:?}", - self.transport, - )); + // TODO: The whole path not just the filename + warn!(path = name, "Unexpected file in archive directory"); } } - Ok(DirEntry { kind, name, .. }) => { - ui::problem(&format!( - "Unexpected file kind in archive directory: {name:?} of kind {kind:?}" - )); - stats.unexpected_files += 1; + Ok(DirEntry { name, .. }) => { + // TODO: The whole path not just the filename + warn!(path = name, "Unexpected file in archive directory"); } - Err(source) => { - ui::problem(&format!("Error listing archive directory: {source:?}")); - stats.io_errors += 1; + Err(err) => { + error!(%err, "Error listing archive directory"); } } } - Ok(stats) + Ok(()) } } diff --git a/src/backup.rs b/src/backup.rs index 3ed2cff3..60f67fb5 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -18,9 +18,11 @@ use std::io::prelude::*; use std::{convert::TryInto, time::Instant}; use itertools::Itertools; +use tracing::error; use crate::blockdir::Address; use crate::io::read_with_retries; +use crate::progress::{Bar, Progress}; use crate::stats::BackupStats; use crate::stitch::IterStitchedIndexHunks; use crate::tree::ReadTree; @@ -64,31 +66,6 @@ impl Default for BackupOptions { // progress_bar.set_bytes_total(source.size()?.file_bytes as u64); // } -#[derive(Default)] -struct ProgressModel { - filename: String, - scanned_file_bytes: u64, - scanned_dirs: usize, - scanned_files: usize, - entries_new: usize, - entries_changed: usize, - entries_unchanged: usize, - entries_deleted: usize, -} - -impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", - self.scanned_dirs, - self.scanned_files, - self.scanned_file_bytes / 1_000_000, - self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, - self.filename - ) - } -} - /// Backup a source directory into a new band in the archive. /// /// Returns statistics about what was copied. @@ -100,52 +77,67 @@ pub fn backup( let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; let mut stats = BackupStats::default(); - let mut view = nutmeg::View::new(ProgressModel::default(), ui::nutmeg_options()); + let bar = Bar::new(); + + let mut scanned_dirs = 0; + let mut scanned_files = 0; + let mut scanned_file_bytes = 0; + let mut entries_new = 0; + let mut entries_changed = 0; + let mut entries_unchanged = 0; let entry_iter = source.iter_entries(Apath::root(), options.exclude.clone())?; for entry_group in entry_iter.chunks(options.max_entries_per_hunk).into_iter() { for entry in entry_group { - view.update(|model| { - model.filename = entry.apath().to_string(); - match entry.kind() { - Kind::Dir => model.scanned_dirs += 1, - Kind::File => model.scanned_files += 1, - _ => (), - } - }); + match entry.kind() { + Kind::Dir => scanned_dirs += 1, + Kind::File => scanned_files += 1, + _ => (), + } match writer.copy_entry(&entry, source) { - Err(e) => { - writeln!(view, "{}", ui::format_error_causes(&e))?; + Err(err) => { + error!(?entry, ?err, "Error copying entry to backup"); stats.errors += 1; continue; } Ok(Some(diff_kind)) => { if options.print_filenames && diff_kind != DiffKind::Unchanged { if options.long_listing { - writeln!( - view, + println!( "{} {} {} {}", diff_kind.as_sigil(), entry.unix_mode(), entry.owner(), entry.apath() - )?; + ); } else { - writeln!(view, "{} {}", diff_kind.as_sigil(), entry.apath())?; + println!("{} {}", diff_kind.as_sigil(), entry.apath()); } } - view.update(|model| match diff_kind { - DiffKind::Changed => model.entries_changed += 1, - DiffKind::New => model.entries_new += 1, - DiffKind::Unchanged => model.entries_unchanged += 1, - DiffKind::Deleted => model.entries_deleted += 1, - }) + match diff_kind { + DiffKind::Changed => entries_changed += 1, + DiffKind::New => entries_new += 1, + DiffKind::Unchanged => entries_unchanged += 1, + // Deletions are not produced at the moment. + DiffKind::Deleted => (), // model.entries_deleted += 1, + } } Ok(_) => {} } if let Some(bytes) = entry.size() { if bytes > 0 { - view.update(|model| model.scanned_file_bytes += bytes) + scanned_file_bytes += bytes; + if !options.print_filenames { + bar.post(Progress::Backup { + filename: entry.apath().to_string(), + scanned_file_bytes, + scanned_dirs, + scanned_files, + entries_new, + entries_changed, + entries_unchanged, + }); + } } } } diff --git a/src/band.rs b/src/band.rs index 4e7fa688..47fe1cfb 100644 --- a/src/band.rs +++ b/src/band.rs @@ -23,6 +23,9 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; +use tracing::error; +#[allow(unused_imports)] +use tracing::warn; use crate::jsonio::{read_json, write_json}; use crate::misc::remove_item; @@ -218,32 +221,20 @@ impl Band { }) } - pub fn validate(&self, stats: &mut ValidateStats) -> Result<()> { + pub fn validate(&self) -> Result<()> { let ListDirNames { mut files, dirs } = self.transport.list_dir_names("").map_err(Error::from)?; if !files.contains(&BAND_HEAD_FILENAME.to_string()) { - ui::problem(&format!("No band head file in {:?}", self.transport)); - stats.missing_band_heads += 1; + error!(band_id = ?self.band_id, "Band head file missing"); } remove_item(&mut files, &BAND_HEAD_FILENAME); remove_item(&mut files, &BAND_TAIL_FILENAME); - - if !files.is_empty() { - ui::problem(&format!( - "Unexpected files in band directory {:?}: {:?}", - self.transport, files - )); - stats.unexpected_files += 1; + for unexpected in files { + warn!(path = ?unexpected, "Unexpected file in band directory"); } - - if dirs != [INDEX_DIR.to_string()] { - ui::problem(&format!( - "Incongruous directories in band directory {:?}: {:?}", - self.transport, dirs - )); - stats.unexpected_files += 1; + for unexpected in dirs.iter().filter(|n| n != &INDEX_DIR) { + warn!(path = ?unexpected, "Unexpected subdirectory in band directory"); } - Ok(()) } } diff --git a/src/bandid.rs b/src/bandid.rs index ec606b04..621ffc96 100644 --- a/src/bandid.rs +++ b/src/bandid.rs @@ -16,12 +16,14 @@ use std::fmt::{self, Write}; use std::str::FromStr; +use serde::Serialize; + use crate::errors::Error; /// Identifier for a band within an archive, eg 'b0001' or 'b0001-0020'. /// /// `BandId`s implement a total ordering `std::cmp::Ord`. -#[derive(Debug, PartialEq, Clone, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, PartialEq, Clone, Eq, PartialOrd, Ord, Hash, Serialize)] pub struct BandId { /// The sequence numbers at each tier. seqs: Vec, diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index 27e7f056..fc1295cd 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -13,13 +13,19 @@ //! Command-line entry point for Conserve backups. +use std::error::Error; use std::io::{BufWriter, Write}; use std::path::PathBuf; +use std::time::Instant; use clap::{Parser, Subcommand}; -use tracing::trace; +use conserve::trace_counter::{global_error_count, global_warn_count}; +use metrics::increment_counter; +#[allow(unused_imports)] +use tracing::{debug, error, info, trace, warn, Level}; use conserve::backup::BackupOptions; +use conserve::ui::termui::TraceTimeStyle; use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; @@ -37,6 +43,18 @@ struct Args { /// Show debug trace to stdout. #[arg(long, short = 'D', global = true)] debug: bool, + + /// Control timestamps prefixes on stderr. + #[arg(long, value_enum, global = true, default_value_t = TraceTimeStyle::None)] + trace_time: TraceTimeStyle, + + /// Append a json formatted log to this file. + #[arg(long, global = true)] + log_json: Option, + + /// Write metrics to this file. + #[arg(long, global = true)] + metrics_json: Option, } #[derive(Debug, Subcommand)] @@ -102,8 +120,6 @@ enum Command { }, /// Delete blocks unreferenced by any index. - /// - /// CAUTION: Do not gc while a backup is underway. Gc { /// Archive to delete from. archive: String, @@ -242,9 +258,15 @@ enum Debug { } enum ExitCode { - Ok = 0, - Failed = 1, - PartialCorruption = 2, + Success = 0, + Failure = 1, + NonFatalErrors = 2, +} + +impl std::process::Termination for ExitCode { + fn report(self) -> std::process::ExitCode { + (self as u8).into() + } } impl Command { @@ -269,7 +291,7 @@ impl Command { }; let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options)?; if !no_stats { - ui::println(&format!("Backup complete.\n{stats}")); + info!("Backup complete.\n{stats}"); } } Command::Debug(Debug::Blocks { archive }) => { @@ -313,7 +335,7 @@ impl Command { }, )?; if !no_stats { - ui::println(&format!("{stats}")); + println!("{stats}"); } } Command::Diff { @@ -347,12 +369,12 @@ impl Command { }, )?; if !no_stats { - ui::println(&format!("{stats}")); + info!(%stats); } } Command::Init { archive } => { Archive::create(open_transport(archive)?)?; - ui::println(&format!("Created new archive in {:?}", &archive)); + debug!("Created new archive in {archive:?}"); } Command::Ls { stos, @@ -400,10 +422,10 @@ impl Command { overwrite: *force_overwrite, long_listing: *long_listing, }; - let stats = restore(&archive, destination, &options)?; + debug!("Restore complete"); if !no_stats { - ui::println(&format!("Restore complete.\n{stats}")); + debug!(%stats); } } Command::Size { @@ -423,28 +445,20 @@ impl Command { .file_bytes }; if *bytes { - ui::println(&format!("{size}")); + println!("{size}"); } else { - ui::println(&conserve::bytes_to_human_mb(size)); + println!("{}", conserve::bytes_to_human_mb(size)); } } - Command::Validate { - archive, - quick, - no_stats, - } => { + Command::Validate { archive, quick, .. } => { let options = ValidateOptions { skip_block_hashes: *quick, }; - let stats = Archive::open(open_transport(archive)?)?.validate(&options)?; - if !no_stats { - println!("{stats}"); - } - if stats.has_problems() { - ui::problem("Archive has some problems."); - return Ok(ExitCode::PartialCorruption); + Archive::open(open_transport(archive)?)?.validate(&options)?; + if global_error_count() > 0 || global_warn_count() > 0 { + warn!("Archive has some problems."); } else { - ui::println("Archive is OK."); + info!("Archive is OK."); } } Command::Versions { @@ -454,7 +468,6 @@ impl Command { sizes, utc, } => { - ui::enable_progress(false); let archive = Archive::open(open_transport(archive)?)?; let options = ShowVersionsOptions { newest_first: *newest, @@ -466,7 +479,7 @@ impl Command { conserve::show_versions(&archive, &options, &mut stdout)?; } } - Ok(ExitCode::Ok) + Ok(ExitCode::Success) } } @@ -484,29 +497,45 @@ fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolic } } -fn main() { +fn main() -> Result { let args = Args::parse(); - ui::enable_progress(!args.no_progress && !args.debug); - if args.debug { - tracing_subscriber::fmt::Subscriber::builder() - .with_max_level(tracing::Level::TRACE) - .init(); - trace!("tracing enabled"); + let start_time = Instant::now(); + if !args.no_progress { + progress::ProgressImpl::Terminal.activate(); } + let trace_level = if args.debug { + Level::TRACE + } else { + Level::INFO + }; + let _flush_guard = ui::termui::enable_tracing(&args.trace_time, trace_level, &args.log_json); + ::metrics::set_recorder(&conserve::metric_recorder::IN_MEMORY) + .expect("Failed to install recorder"); + increment_counter!("conserve.start"); let result = args.command.run(); + metric_recorder::emit_to_trace(); + debug!(elapsed = ?start_time.elapsed()); + let error_count = global_error_count(); + let warn_count = global_warn_count(); + if let Some(metrics_json_path) = args.metrics_json { + metric_recorder::write_json_metrics(&metrics_json_path)?; + } match result { - Err(ref e) => { - ui::show_error(e); - // // TODO: Perhaps always log the traceback to a log file. - // if let Some(bt) = e.backtrace() { - // if std::env::var("RUST_BACKTRACE") == Ok("1".to_string()) { - // println!("{}", bt); - // } - // } - // Avoid Rust redundantly printing the error. - std::process::exit(ExitCode::Failed as i32) + Err(err) => { + error!("{err}"); + let mut err: &dyn Error = &err; + while let Some(source) = err.source() { + error!("caused by: {source}"); + err = source; + } + debug!(error_count, warn_count,); + Ok(ExitCode::Failure) + } + Ok(ExitCode::Success) if error_count > 0 || warn_count > 0 => { + debug!(error_count, warn_count,); + Ok(ExitCode::NonFatalErrors) } - Ok(code) => std::process::exit(code as i32), + Ok(exit_code) => Ok(exit_code), } } diff --git a/src/blockdir.rs b/src/blockdir.rs index d125f00f..931477af 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -25,20 +25,23 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::io; use std::path::Path; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Instant; +use ::metrics::{counter, histogram, increment_counter}; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; -use nutmeg::models::UnboundedModel; use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use thousands::Separable; +#[allow(unused_imports)] +use tracing::{debug, error, info, warn}; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; use crate::kind::Kind; -use crate::stats::{BackupStats, Sizes, ValidateStats}; +use crate::progress::{Bar, Progress}; +use crate::stats::{BackupStats, Sizes}; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, ListDirNames, Transport}; use crate::*; @@ -112,19 +115,23 @@ impl BlockDir { pub(crate) fn compress_and_store(&mut self, in_buf: &[u8], hash: &BlockHash) -> Result { // TODO: Move this to a BlockWriter, which can hold a reusable buffer. let mut compressor = Compressor::new(); + let uncomp_len = in_buf.len() as u64; let compressed = compressor.compress(in_buf)?; let comp_len: u64 = compressed.len().try_into().unwrap(); let hex_hash = hash.to_string(); let relpath = block_relpath(hash); self.transport.create_dir(subdir_relpath(&hex_hash))?; + increment_counter!("conserve.block.writes"); + counter!("conserve.block.write_uncompressed_bytes", uncomp_len); + histogram!("conserve.block.write_uncompressed_bytes", uncomp_len as f64); + counter!("conserve.block.write_compressed_bytes", comp_len); + histogram!("conserve.block.write_compressed_bytes", comp_len as f64); self.transport .write_file(&relpath, compressed) .or_else(|io_err| { if io_err.kind() == io::ErrorKind::AlreadyExists { // Perhaps it was simultaneously created by another thread or process. - ui::problem(&format!( - "Unexpected late detection of existing block {hex_hash:?}" - )); + debug!("Unexpected late detection of existing block {hex_hash:?}"); Ok(()) } else { Err(Error::WriteBlock { @@ -142,11 +149,16 @@ impl BlockDir { stats: &mut BackupStats, ) -> Result { let hash = self.hash_bytes(block_data); + let len = block_data.len() as u64; if self.contains(&hash)? { + increment_counter!("conserve.block.matches"); stats.deduplicated_blocks += 1; - stats.deduplicated_bytes += block_data.len() as u64; + counter!("conserve.block.matched_bytes", len); + stats.deduplicated_bytes += len; } else { + let start = Instant::now(); let comp_len = self.compress_and_store(block_data, &hash)?; + histogram!("conserve.block.compress_and_store_seconds", start.elapsed()); stats.written_blocks += 1; stats.uncompressed_bytes += block_data.len() as u64; stats.compressed_bytes += comp_len; @@ -204,7 +216,7 @@ impl BlockDir { if dirname.len() == SUBDIR_NAME_CHARS { true } else { - ui::problem(&format!("Unexpected subdirectory in blockdir: {dirname:?}")); + warn!("Unexpected subdirectory in blockdir: {dirname:?}"); false } }); @@ -219,14 +231,14 @@ impl BlockDir { .map(move |subdir_name| transport.iter_dir_entries(&subdir_name)) .filter_map(|iter_or| { if let Err(ref err) = iter_or { - ui::problem(&format!("Error listing block directory: {:?}", &err)); + error!(%err, "Error listing block subdirectory"); } iter_or.ok() }) .flatten() .filter_map(|iter_or| { if let Err(ref err) = iter_or { - ui::problem(&format!("Error listing block subdirectory: {:?}", &err)); + error!(%err, "Error listing block subdirectory"); } iter_or.ok() }) @@ -239,20 +251,26 @@ impl BlockDir { /// Return all the blocknames in the blockdir, in arbitrary order. pub fn block_names(&self) -> Result> { - let progress = nutmeg::View::new("List blocks", ui::nutmeg_options()); - progress.update(|_| ()); + // TODO: Report errors Ok(self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok())) } - /// Return all the blocknames in the blockdir. + /// Return all the blocknames in the blockdir, while showing progress. pub fn block_names_set(&self) -> Result> { - let progress = nutmeg::View::new(UnboundedModel::new("List blocks"), ui::nutmeg_options()); + // TODO: We could estimate time remaining by accounting for how + // many prefixes are present and how many have been read. + // TODO: Read prefixes in parallel. + let bar = Bar::new(); Ok(self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok()) - .inspect(|_| progress.update(|model| model.increment(1))) + .enumerate() + .map(|(count, hash)| { + bar.post(Progress::ListBlocks { count }); + hash + }) .collect()) } @@ -260,68 +278,39 @@ impl BlockDir { /// /// Return a dict describing which blocks are present, and the length of their uncompressed /// data. - pub fn validate(&self, stats: &mut ValidateStats) -> Result> { + pub fn validate(&self) -> Result> { // TODO: In the top-level directory, no files or directories other than prefix // directories of the right length. // TODO: Test having a block with the right compression but the wrong contents. - ui::println("Count blocks..."); + debug!("Start list blocks"); let blocks = self.block_names_set()?; - crate::ui::println(&format!( - "Check {} blocks...", - blocks.len().separate_with_commas() - )); - stats.block_read_count = blocks.len().try_into().unwrap(); - struct ProgressModel { - total_blocks: usize, - blocks_done: usize, - bytes_done: usize, - start: Instant, - } - impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Check block {}/{}: {} done, {} MB checked, {} remaining", - self.blocks_done, - self.total_blocks, - nutmeg::percent_done(self.blocks_done, self.total_blocks), - self.bytes_done / 1_000_000, - nutmeg::estimate_remaining(&self.start, self.blocks_done, self.total_blocks) - ) - } - } - let progress_bar = nutmeg::View::new( - ProgressModel { - total_blocks: blocks.len(), - blocks_done: 0, - bytes_done: 0, - start: Instant::now(), - }, - ui::nutmeg_options(), - ); - // Make a vec of Some(usize) if the block could be read, or None if it - // failed, where the usize gives the uncompressed data size. - let results: Vec> = blocks + let total_blocks = blocks.len(); + debug!("Check {total_blocks} blocks"); + let blocks_done = AtomicUsize::new(0); + let bytes_done = AtomicU64::new(0); + let start = Instant::now(); + let task = Bar::new(); + let block_lens = blocks .into_par_iter() - // .into_iter() - .map(|hash| { - let r = self - .get_block_content(&hash) - .map(|(bytes, _sizes)| (hash, bytes.len())) - .ok(); - let bytes = r.as_ref().map(|x| x.1).unwrap_or_default(); - progress_bar.update(|model| { - model.blocks_done += 1; - model.bytes_done += bytes - }); - r + .flat_map(|hash| match self.get_block_content(&hash) { + Ok((bytes, _sizes)) => { + let len = bytes.len(); + let len64 = len as u64; + task.post(Progress::ValidateBlocks { + blocks_done: blocks_done.fetch_add(1, Ordering::Relaxed) + 1, + total_blocks, + bytes_done: bytes_done.fetch_add(len64, Ordering::Relaxed) + len64, + start, + }); + Some((hash, len)) + } + Err(err) => { + error!(%err, %hash, "Error reading block content"); + None + } }) .collect(); - stats.block_error_count += results.iter().filter(|o| o.is_none()).count(); - let len_map: HashMap = results - .into_iter() - .flatten() // keep only Some values - .collect(); - Ok(len_map) + Ok(block_lens) } /// Return the entire contents of the block. @@ -330,6 +319,8 @@ impl BlockDir { pub fn get_block_content(&self, hash: &BlockHash) -> Result<(Vec, Sizes)> { // TODO: Reuse decompressor buffer. // TODO: Reuse read buffer. + // TODO: Most importantly, cache decompressed blocks! + increment_counter!("conserve.block.read"); let mut decompressor = Decompressor::new(); let block_relpath = block_relpath(hash); let compressed_bytes = @@ -346,10 +337,7 @@ impl BlockDir { decompressed_bytes, )); if actual_hash != *hash { - ui::problem(&format!( - "Block file {:?} has actual decompressed hash {}", - &block_relpath, actual_hash - )); + error!("Block file {block_relpath:?} has actual decompressed hash {actual_hash}"); return Err(Error::BlockCorrupt { hash: hash.to_string(), actual_hash: actual_hash.to_string(), diff --git a/src/errors.rs b/src/errors.rs index 962dde23..0ca9db7c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -13,17 +13,38 @@ //! Conserve error types. +use std::io; use std::path::PathBuf; +use serde::{self, ser::SerializeStruct, Serialize, Serializer}; use thiserror::Error; use crate::blockdir::Address; use crate::*; -type IOError = std::io::Error; +fn serialize_io_error(err: &io::Error, s: S) -> std::result::Result +where + S: Serializer, +{ + let mut t = s.serialize_struct("io::Error", 2)?; + t.serialize_field("os_error", &err.raw_os_error())?; + t.serialize_field("message", &err.to_string())?; + t.end() +} + +fn serialize_generic_error(err: &E, s: S) -> std::result::Result +where + E: std::error::Error, + S: Serializer, +{ + let mut t = s.serialize_struct("Error", 1)?; + t.serialize_field("message", &err.to_string())?; + t.end() +} /// Conserve specific error. -#[derive(Debug, Error)] +#[non_exhaustive] +#[derive(Debug, Error, Serialize)] pub enum Error { #[error("Block file {hash:?} corrupt; actual hash {actual_hash:?}")] BlockCorrupt { hash: String, actual_hash: String }, @@ -31,20 +52,47 @@ pub enum Error { #[error("{address:?} extends beyond decompressed block length {actual_len:?}")] AddressTooLong { address: Address, actual_len: usize }, + // TODO: Merge with AddressTooLong + #[error( + "block {block_hash} actual length is {actual_len} but indexes reference {referenced_len}" + )] + ShortBlock { + block_hash: BlockHash, + actual_len: usize, + referenced_len: u64, + }, + #[error("Failed to write block {hash:?}")] - WriteBlock { hash: String, source: IOError }, + WriteBlock { + hash: String, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to read block {hash:?}")] - ReadBlock { hash: String, source: IOError }, + ReadBlock { + hash: String, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, + + #[error("Block {block_hash} is missing")] + BlockMissing { block_hash: BlockHash }, #[error("Failed to list block files")] - ListBlocks { source: IOError }, + ListBlocks { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Not a Conserve archive")] NotAnArchive {}, #[error("Failed to read archive header")] - ReadArchiveHeader { source: std::io::Error }, + ReadArchiveHeader { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error( "Archive version {:?} is not supported by Conserve {}", @@ -72,17 +120,32 @@ pub enum Error { InvalidVersion { version: String }, #[error("Failed to create band")] - CreateBand { source: std::io::Error }, + CreateBand { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, + + #[error("Band {band_id} head file missing")] + BandHeadMissing { band_id: BandId }, #[error("Failed to create block directory")] - CreateBlockDir { source: std::io::Error }, + CreateBlockDir { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to create archive directory")] - CreateArchiveDirectory { source: std::io::Error }, + CreateArchiveDirectory { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Band {} is incomplete", band_id)] BandIncomplete { band_id: BandId }, + #[error("Duplicated band directory for {band_id}")] + DuplicateBandDirectory { band_id: BandId }, + #[error( "Can't delete blocks because the last band ({}) is incomplete and may be in use", band_id @@ -98,87 +161,144 @@ pub enum Error { #[error(transparent)] ParseGlob { #[from] + #[serde(serialize_with = "serialize_generic_error")] source: globset::Error, }, #[error("Failed to write index hunk {:?}", path)] - WriteIndex { path: String, source: IOError }, + WriteIndex { + path: String, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to read index hunk {:?}", path)] - ReadIndex { path: String, source: IOError }, + ReadIndex { + path: String, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to serialize index")] - SerializeIndex { source: serde_json::Error }, + SerializeIndex { + #[serde(serialize_with = "serialize_generic_error")] + source: serde_json::Error, + }, #[error("Failed to deserialize index hunk {:?}", path)] DeserializeIndex { path: String, + #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Failed to write metadata file {:?}", path)] WriteMetadata { path: String, - source: std::io::Error, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, }, #[error("Failed to deserialize json from {:?}", path)] DeserializeJson { path: PathBuf, + #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Failed to serialize json to {:?}", path)] SerializeJson { path: String, + #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Metadata file not found: {:?}", path)] MetadataNotFound { path: String, - source: std::io::Error, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, }, #[error("Failed to list bands")] - ListBands { source: std::io::Error }, + ListBands { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to read source file {:?}", path)] ReadSourceFile { path: PathBuf, - source: std::io::Error, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, }, #[error("Failed to read source tree {:?}", path)] - ListSourceTree { path: PathBuf, source: IOError }, + ListSourceTree { + path: PathBuf, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to store file {:?}", apath)] - StoreFile { apath: Apath, source: IOError }, + StoreFile { + apath: Apath, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to restore {:?}", path)] - Restore { path: PathBuf, source: IOError }, + Restore { + path: PathBuf, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to restore modification time on {:?}", path)] - RestoreModificationTime { path: PathBuf, source: IOError }, + RestoreModificationTime { + path: PathBuf, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Failed to delete band {}", band_id)] - BandDeletion { band_id: BandId, source: IOError }, + BandDeletion { + band_id: BandId, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + }, #[error("Unsupported URL scheme {:?}", scheme)] UrlScheme { scheme: String }, + #[error("Failed to serialize problem")] + SerializeError { + #[serde(serialize_with = "serialize_generic_error")] + source: serde_json::Error, + }, + + #[error("Unexpected file {path:?} in archive directory")] + UnexpectedFile { path: String }, + /// Generic IO error. #[error(transparent)] IOError { #[from] - source: IOError, + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, }, #[error("Failed to set owner of {path:?}")] - SetOwner { source: IOError, path: PathBuf }, + SetOwner { + #[serde(serialize_with = "serialize_io_error")] + source: io::Error, + path: PathBuf, + }, #[error(transparent)] SnapCompressionError { + // TODO: Maybe say in which file, etc. + #[serde(serialize_with = "serialize_generic_error")] #[from] source: snap::Error, }, diff --git a/src/index.rs b/src/index.rs index 864a6082..a81a8f53 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -20,6 +20,9 @@ use std::path::Path; use std::sync::Arc; use std::vec; +use metrics::{counter, increment_counter}; +use tracing::error; + use crate::compress::snappy::{Compressor, Decompressor}; use crate::kind::Kind; use crate::owner::Owner; @@ -340,9 +343,7 @@ impl Iterator for IndexHunkIter { Ok(Some(entries)) => entries, Err(err) => { self.stats.errors += 1; - ui::problem(&format!( - "Error reading index hunk {hunk_number:?}: {err:?} " - )); + error!("Error reading index hunk {hunk_number:?}: {err}"); continue; } }; @@ -382,10 +383,10 @@ impl IndexHunkIter { } fn read_next_hunk(&mut self) -> Result>> { - let path = &hunk_relpath(self.next_hunk_number); + let path = hunk_relpath(self.next_hunk_number); // Whether we succeed or fail, don't try to read this hunk again. self.next_hunk_number += 1; - let compressed_bytes = match self.transport.read_file(path) { + let compressed_bytes = match self.transport.read_file(&path) { Ok(b) => b, Err(err) if err.kind() == io::ErrorKind::NotFound => { // TODO: Cope with one hunk being missing, while there are still @@ -400,15 +401,25 @@ impl IndexHunkIter { }); } }; + increment_counter!("conserve.index.read.hunks"); self.stats.index_hunks += 1; + counter!( + "conserve.index.read.compressed_bytes", + compressed_bytes.len() as u64 + ); self.stats.compressed_index_bytes += compressed_bytes.len() as u64; let index_bytes = self.decompressor.decompress(&compressed_bytes)?; + counter!( + "conserve.index.read.decompressed_bytes", + index_bytes.len() as u64 + ); self.stats.uncompressed_index_bytes += index_bytes.len() as u64; let entries: Vec = serde_json::from_slice(index_bytes).map_err(|source| Error::DeserializeIndex { path: path.clone(), source, })?; + counter!("conserve.index.read.entries", entries.len() as u64); if entries.is_empty() { // It's legal, it's just weird - and it can be produced by some old Conserve versions. } diff --git a/src/lib.rs b/src/lib.rs index 64a043f4..982a494b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -12,7 +12,6 @@ //! Conserve backup system. -// Conserve implementation modules. pub mod apath; pub mod archive; pub mod backup; @@ -32,8 +31,10 @@ mod jsonio; pub mod kind; pub mod live_tree; mod merge; -pub(crate) mod misc; +pub mod metric_recorder; +pub mod misc; pub mod owner; +pub mod progress; pub mod restore; pub mod show; pub mod stats; @@ -41,12 +42,13 @@ mod stitch; mod stored_file; mod stored_tree; pub mod test_fixtures; +pub mod trace_counter; pub mod transport; mod tree; pub mod ui; pub mod unix_mode; pub mod unix_time; -mod validate; +pub mod validate; pub use crate::apath::Apath; pub use crate::archive::Archive; @@ -69,7 +71,7 @@ pub use crate::merge::{MergeTrees, MergedEntryKind}; pub use crate::misc::bytes_to_human_mb; pub use crate::restore::{restore, RestoreOptions, RestoreTree}; pub use crate::show::{show_diff, show_versions, ShowVersionsOptions}; -pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; +pub use crate::stats::{BackupStats, DeleteStats, RestoreStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; diff --git a/src/live_tree.rs b/src/live_tree.rs index 41b54531..015b213d 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -18,6 +18,8 @@ use std::fs; use std::io::ErrorKind; use std::path::{Path, PathBuf}; +use tracing::{error, warn}; + use crate::owner::Owner; use crate::stats::LiveTreeIterStats; use crate::unix_mode::UnixMode; @@ -211,8 +213,8 @@ impl Iter { let dir_path = parent_apath.below(&self.root_path); let dir_iter = match fs::read_dir(&dir_path) { Ok(i) => i, - Err(e) => { - ui::problem(&format!("Error reading directory {:?}: {}", &dir_path, e)); + Err(err) => { + error!("Error reading directory {dir_path:?}: {err}"); return; } }; @@ -220,11 +222,8 @@ impl Iter { for dir_entry in dir_iter { let dir_entry = match dir_entry { Ok(dir_entry) => dir_entry, - Err(e) => { - ui::problem(&format!( - "Error reading next entry from directory {:?}: {}", - &dir_path, e - )); + Err(err) => { + error!("Error reading next entry from directory {dir_path:?}: {err}"); continue; } }; @@ -232,9 +231,7 @@ impl Iter { let child_name = match child_osstr.to_str() { Some(c) => c, None => { - ui::problem(&format!( - "Couldn't decode filename {child_osstr:?} in {dir_path:?}", - )); + error!("Couldn't decode filename {child_osstr:?} in {dir_path:?}",); continue; } }; @@ -248,9 +245,7 @@ impl Iter { let ft = match dir_entry.file_type() { Ok(ft) => ft, Err(e) => { - ui::problem(&format!( - "Error getting type of {child_apath:?} during iteration: {e}" - )); + error!("Error getting type of {child_apath:?} during iteration: {e}"); continue; } }; @@ -261,9 +256,7 @@ impl Iter { Ok(true) => continue, Ok(false) => (), Err(e) => { - ui::problem(&format!( - "Error checking CACHEDIR.TAG in {dir_entry:?}: {e}" - )); + error!("Error checking CACHEDIR.TAG in {dir_entry:?}: {e}"); } } } @@ -275,14 +268,10 @@ impl Iter { ErrorKind::NotFound => { // Fairly harmless, and maybe not even worth logging. Just a race // between listing the directory and looking at the contents. - ui::problem(&format!( - "File disappeared during iteration: {child_apath:?}: {e}" - )); + warn!("File disappeared during iteration: {child_apath:?}: {e}"); } _ => { - ui::problem(&format!( - "Failed to read source metadata from {child_apath:?}: {e}" - )); + error!("Failed to read source metadata from {child_apath:?}: {e}"); self.stats.metadata_error += 1; } }; @@ -296,18 +285,14 @@ impl Iter { let t = match dir_path.join(dir_entry.file_name()).read_link() { Ok(t) => t, Err(e) => { - ui::problem(&format!( - "Failed to read target of symlink {child_apath:?}: {e}" - )); + error!("Failed to read target of symlink {child_apath:?}: {e}"); continue; } }; match t.into_os_string().into_string() { Ok(t) => Some(t), Err(e) => { - ui::problem(&format!( - "Failed to decode target of symlink {child_apath:?}: {e:?}" - )); + error!("Failed to decode target of symlink {child_apath:?}: {e:?}"); continue; } } diff --git a/src/metric_recorder.rs b/src/metric_recorder.rs new file mode 100644 index 00000000..663f50d0 --- /dev/null +++ b/src/metric_recorder.rs @@ -0,0 +1,152 @@ +// Copyright 2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! A metrics recorder that just keeps atomic values in memory, +//! so they can be logged or inspected at the end of the process, or potentially +//! earlier. + +use std::collections::BTreeMap; +use std::fs::OpenOptions; +use std::path::Path; +use std::sync::atomic::AtomicU64; +use std::sync::Mutex; +use std::sync::{atomic::Ordering, Arc}; + +use ::metrics::{ + Counter, Gauge, Histogram, HistogramFn, Key, KeyName, Recorder, SharedString, Unit, +}; +use itertools::Itertools; +use lazy_static::lazy_static; +use metrics_util::registry::{Registry, Storage}; +use metrics_util::Summary; +use serde_json::json; +use tracing::debug; + +use crate::{Error, Result}; + +lazy_static! { + static ref REGISTRY: Registry = Registry::new(SummaryStorage::new()); +} + +pub struct InMemory {} +pub static IN_MEMORY: InMemory = InMemory {}; + +impl Recorder for InMemory { + fn describe_counter(&self, _key: KeyName, _unit: Option, _description: SharedString) { + todo!() + } + + fn describe_gauge(&self, _key: KeyName, _unit: Option, _description: SharedString) { + todo!() + } + + fn describe_histogram(&self, __key: KeyName, __unit: Option, _description: SharedString) { + todo!() + } + + fn register_counter(&self, key: &Key) -> Counter { + REGISTRY.get_or_create_counter(key, |c| Counter::from_arc(Arc::clone(c))) + } + + fn register_gauge(&self, _key: &Key) -> Gauge { + todo!() + } + + fn register_histogram(&self, key: &Key) -> Histogram { + REGISTRY.get_or_create_histogram(key, |g| Histogram::from_arc(Arc::clone(g))) + } +} + +pub fn counter_values() -> BTreeMap { + REGISTRY + .get_counter_handles() + .into_iter() + .map(|(key, counter)| (key.name().to_owned(), counter.load(Ordering::Relaxed))) + .collect() +} + +pub fn emit_to_trace() { + for (counter_name, count) in counter_values() { + debug!(counter_name, count); + } + for (histogram_name, histogram) in REGISTRY + .get_histogram_handles() + .into_iter() + .sorted_by_key(|(k, _v)| k.clone()) + { + let summary = histogram.0.lock().unwrap(); + debug!( + histogram = histogram_name.name(), + p10 = summary.quantile(0.1), + p50 = summary.quantile(0.5), + p90 = summary.quantile(0.9), + p99 = summary.quantile(0.99), + p100 = summary.quantile(1.0), + ); + } +} + +/// Like AtomicStorage but using a Summary. +struct SummaryStorage {} + +impl SummaryStorage { + const fn new() -> Self { + SummaryStorage {} + } +} + +impl Storage for SummaryStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, _key: &K) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _: &K) -> Self::Gauge { + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, _: &K) -> Self::Histogram { + Arc::new(SummaryHistogram::new()) + } +} + +struct SummaryHistogram(Mutex); + +impl HistogramFn for SummaryHistogram { + fn record(&self, value: f64) { + self.0.lock().unwrap().add(value) + } +} + +impl SummaryHistogram { + fn new() -> Self { + SummaryHistogram(Mutex::new(Summary::with_defaults())) + } +} + +pub fn write_json_metrics(path: &Path) -> Result<()> { + let f = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + let j = json!( { + "counters": counter_values(), + }); + serde_json::to_writer_pretty(f, &j).map_err(|source| Error::SerializeJson { + path: path.to_string_lossy().to_string(), + source, + }) +} diff --git a/src/misc.rs b/src/misc.rs index c36e3ed4..03a323d5 100644 --- a/src/misc.rs +++ b/src/misc.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -13,6 +13,10 @@ //! Generally useful functions. +use std::time::Duration; + +use crate::stats::Sizes; + /// Remove and return an item from a vec, if it's present. pub(crate) fn remove_item>(v: &mut Vec, item: &U) { // Remove this when it's stabilized in std: @@ -44,3 +48,80 @@ pub(crate) fn zero_u32(a: &u32) -> bool { pub(crate) fn zero_u64(a: &u64) -> bool { *a == 0 } + +#[allow(unused)] +pub(crate) fn compression_percent(s: &Sizes) -> i64 { + if s.uncompressed > 0 { + 100i64 - (100 * s.compressed / s.uncompressed) as i64 + } else { + 0 + } +} + +pub fn duration_to_hms(d: Duration) -> String { + let elapsed_secs = d.as_secs(); + if elapsed_secs >= 3600 { + format!( + "{:2}:{:02}:{:02}", + elapsed_secs / 3600, + (elapsed_secs / 60) % 60, + elapsed_secs % 60 + ) + } else { + format!(" {:2}:{:02}", (elapsed_secs / 60) % 60, elapsed_secs % 60) + } +} + +#[allow(unused)] +pub(crate) fn mbps_rate(bytes: u64, elapsed: Duration) -> f64 { + let secs = elapsed.as_secs() as f64 + f64::from(elapsed.subsec_millis()) / 1000.0; + if secs > 0.0 { + bytes as f64 / secs / 1e6 + } else { + 0f64 + } +} + +/// Describe the compression ratio: higher is better. +#[allow(unused)] +pub(crate) fn compression_ratio(s: &Sizes) -> f64 { + if s.compressed > 0 { + s.uncompressed as f64 / s.compressed as f64 + } else { + 0f64 + } +} + +/// Adds `Result::inspect_err` which is not yet stabilized. +pub(crate) trait ResultExt { + type T; + type E; + fn our_inspect_err(self, f: F) -> Self; +} + +impl ResultExt for std::result::Result { + type T = T; + type E = E; + + #[inline] + fn our_inspect_err(self, f: F) -> Self { + if let Err(ref e) = self { + f(e); + } + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_compression_ratio() { + let ratio = compression_ratio(&Sizes { + compressed: 2000, + uncompressed: 4000, + }); + assert_eq!(format!("{ratio:3.1}x"), "2.0x"); + } +} diff --git a/src/progress.rs b/src/progress.rs new file mode 100644 index 00000000..92b299b3 --- /dev/null +++ b/src/progress.rs @@ -0,0 +1,153 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Generic progress bar indications. + +// static PROGRESS_IMPL; + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::RwLock; +use std::time::Instant; + +static IMPL: RwLock = RwLock::new(ProgressImpl::Null); + +static NEXT_TASK_ID: AtomicUsize = AtomicUsize::new(0); + +pub(crate) mod term; + +/// How to show progress bars? +#[derive(Debug, Clone, Copy)] +pub enum ProgressImpl { + Null, + Terminal, +} + +impl ProgressImpl { + /// Make this the selected way to show progress bars. + pub fn activate(self) { + *IMPL.write().expect("locked progress impl") = self + } + + fn remove_bar(&mut self, task: &mut Bar) { + match self { + ProgressImpl::Null => (), + ProgressImpl::Terminal => term::remove_bar(task.bar_id), + } + } + + fn add_bar(&mut self) -> Bar { + let bar_id = assign_new_bar_id(); + match self { + ProgressImpl::Null => (), + ProgressImpl::Terminal => term::add_bar(bar_id), + } + Bar { bar_id } + } + + fn post(&self, task: &Bar, progress: Progress) { + match self { + ProgressImpl::Null => (), + ProgressImpl::Terminal => term::update_bar(task.bar_id, progress), + } + } +} + +fn assign_new_bar_id() -> usize { + NEXT_TASK_ID.fetch_add(1, Ordering::Relaxed) +} + +/// State of progress on one bar. +#[derive(Clone)] +pub enum Progress { + None, + Backup { + filename: String, + scanned_file_bytes: u64, + scanned_dirs: usize, + scanned_files: usize, + entries_new: usize, + entries_changed: usize, + entries_unchanged: usize, + }, + DeleteBands { + bands_done: usize, + total_bands: usize, + }, + DeleteBlocks { + blocks_done: usize, + total_blocks: usize, + }, + ListBlocks { + count: usize, + }, + MeasureUnreferenced { + blocks_done: usize, + blocks_total: usize, + }, + MeasureTree { + files: usize, + total_bytes: u64, + }, + ReferencedBlocks { + bands_started: usize, + total_bands: usize, + references_found: usize, + start: Instant, + }, + Restore { + filename: String, + bytes_done: u64, + }, + ValidateBands { + total_bands: usize, + bands_done: usize, + start: Instant, + }, + ValidateBlocks { + blocks_done: usize, + total_blocks: usize, + bytes_done: u64, + start: Instant, + }, +} + +/// A transient progress task. The UI may draw these as some kind of +/// progress bar. +#[derive(Debug)] +pub struct Bar { + /// An opaque unique ID for each concurrent task. + bar_id: usize, +} + +impl Bar { + #[must_use] + pub fn new() -> Self { + IMPL.write().expect("lock progress impl").add_bar() + } + + pub fn post(&self, progress: Progress) { + IMPL.read().unwrap().post(self, progress) + } +} + +impl Default for Bar { + fn default() -> Self { + Bar::new() + } +} + +impl Drop for Bar { + fn drop(&mut self) { + IMPL.write().expect("lock progress impl").remove_bar(self) + } +} diff --git a/src/progress/term.rs b/src/progress/term.rs new file mode 100644 index 00000000..f9d4017a --- /dev/null +++ b/src/progress/term.rs @@ -0,0 +1,207 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +use std::io; + +use itertools::Itertools; +use lazy_static::lazy_static; +use nutmeg::estimate_remaining; +use thousands::Separable; + +use super::*; + +lazy_static! { + /// A global Nutmeg view. + /// + /// This is global to reflect that there is globally one stdout/stderr: + /// this object manages it. + static ref NUTMEG_VIEW: nutmeg::View = + nutmeg::View::new( + MultiModel::new(), + nutmeg::Options::new() + .destination(nutmeg::Destination::Stderr) + ); +} + +pub(super) fn add_bar(bar_id: usize) { + NUTMEG_VIEW.update(|model| model.add_bar(bar_id)); +} + +/// Show progress on the global terminal progress bar, +/// or clear the bar if it's [Progress::None]. +pub(super) fn update_bar(bar_id: usize, progress: Progress) { + NUTMEG_VIEW.update(|model| model.update_bar(bar_id, progress)); +} + +pub(super) fn remove_bar(bar_id: usize) { + let removed_last = NUTMEG_VIEW.update(|model| model.remove_bar(bar_id)); + if removed_last { + NUTMEG_VIEW.clear(); + } +} + +/// A stack of multiple Progress objects, each identified by an integer id. +/// +/// Each entry corresponds to one progress::Bar in the abstract interface. +struct MultiModel(Vec<(usize, Progress)>); + +impl MultiModel { + const fn new() -> Self { + MultiModel(Vec::new()) + } + + fn add_bar(&mut self, bar_id: usize) { + assert!( + !self.0.iter().any(|x| x.0 == bar_id), + "task_id should not be already present" + ); + self.0.push((bar_id, Progress::None)); + } + + fn update_bar(&mut self, bar_id: usize, progress: Progress) { + let pos = self + .0 + .iter() + .position(|x| x.0 == bar_id) + .expect("task_id should be present"); + self.0[pos].1 = progress; + } + + fn remove_bar(&mut self, bar_id: usize) -> bool { + self.0.retain(|(id, _)| *id != bar_id); + self.0.is_empty() + } +} + +impl nutmeg::Model for MultiModel { + fn render(&mut self, width: usize) -> String { + self.0.iter_mut().map(|(_id, p)| p.render(width)).join("\n") + } +} + +impl nutmeg::Model for Progress { + fn render(&mut self, _width: usize) -> String { + match self { + Progress::None => String::new(), + Progress::Backup { + filename, + scanned_file_bytes, + scanned_dirs, + scanned_files, + entries_new, + entries_changed, + entries_unchanged, + } => format!( + "\ + Scanned {dirs} directories, {files} files, {mb} MB\n\ + {new} new entries, {changed} changed, {unchanged} unchanged\n\ + {filename}", + dirs = scanned_dirs.separate_with_commas(), + files = scanned_files.separate_with_commas(), + mb = (*scanned_file_bytes / 1_000_000).separate_with_commas(), + new = entries_new.separate_with_commas(), + changed = entries_changed.separate_with_commas(), + unchanged = entries_unchanged.separate_with_commas(), + ), + Progress::DeleteBands { + bands_done, + total_bands, + } => format!( + "Delete bands: {}/{}...", + bands_done.separate_with_commas(), + total_bands.separate_with_commas(), + ), + Progress::DeleteBlocks { + blocks_done, + total_blocks, + } => format!( + "Delete blocks: {}/{}...", + blocks_done.separate_with_commas(), + total_blocks.separate_with_commas(), + ), + Progress::ListBlocks { count } => format!( + "List blocks: {count}...", + count = count.separate_with_commas(), + ), + Progress::MeasureTree { files, total_bytes } => format!( + "Measuring... {} files, {} MB", + files.separate_with_commas(), + (*total_bytes / 1_000_000).separate_with_commas() + ), + Progress::MeasureUnreferenced { + blocks_done, + blocks_total, + } => format!( + "Measure unreferenced blocks: {}/{}...", + blocks_done.separate_with_commas(), + blocks_total.separate_with_commas(), + ), + Progress::ReferencedBlocks { + references_found, + bands_started, + total_bands, + start, + } => format!( + "Find referenced blocks: {found} in {bands_started}/{total_bands} bands, {eta} remaining...", + found = references_found.separate_with_commas(), + eta = estimate_remaining(start, *bands_started, *total_bands), + ), + Progress::Restore { + filename, + bytes_done, + } => format!( + "Restoring: {mb} MB\n{filename}", + mb = *bytes_done / 1_000_000, + ), + Progress::ValidateBlocks { + blocks_done, + total_blocks, + bytes_done, + start, + } => { + format!( + "Check block {}/{}: {} done, {} MB checked, {} remaining", + blocks_done.separate_with_commas(), + total_blocks.separate_with_commas(), + nutmeg::percent_done(*blocks_done, *total_blocks), + (*bytes_done / 1_000_000).separate_with_commas(), + nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) + ) + } + Progress::ValidateBands { + total_bands, + bands_done, + start, + } => format!( + "Check index {}/{}, {} done, {} remaining", + bands_done, + total_bands, + nutmeg::percent_done(*bands_done, *total_bands), + nutmeg::estimate_remaining(start, *bands_done, *total_bands) + ), + } + } +} + +pub(crate) struct WriteToNutmeg(); + +impl io::Write for WriteToNutmeg { + fn write(&mut self, buf: &[u8]) -> io::Result { + NUTMEG_VIEW.message_bytes(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/restore.rs b/src/restore.rs index a441dfc1..0ef78e0b 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -21,10 +21,13 @@ use std::{fs, time::Instant}; use filetime::set_file_handle_times; #[cfg(unix)] use filetime::set_symlink_file_times; +#[allow(unused_imports)] +use tracing::{error, warn}; use crate::band::BandSelectionPolicy; use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; +use crate::progress::{Bar, Progress}; use crate::stats::RestoreStats; use crate::unix_mode::UnixMode; use crate::unix_time::UnixTime; @@ -57,21 +60,6 @@ impl Default for RestoreOptions { } } -struct ProgressModel { - filename: String, - bytes_done: u64, -} - -impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Restoring: {} MB\n{}", - self.bytes_done / 1_000_000, - self.filename - ) - } -} - /// Restore a selected version, or by default the latest, to a destination directory. pub fn restore( archive: &Archive, @@ -85,13 +73,8 @@ pub fn restore( RestoreTree::create(destination_path) }?; let mut stats = RestoreStats::default(); - let progress_bar = nutmeg::View::new( - ProgressModel { - filename: String::new(), - bytes_done: 0, - }, - ui::nutmeg_options(), - ); + let mut bytes_done = 0; + let bar = Bar::new(); let start = Instant::now(); // // This causes us to walk the source tree twice, which is probably an acceptable option // // since it's nice to see realistic overall progress. We could keep all the entries @@ -111,18 +94,18 @@ pub fn restore( for entry in entry_iter { if options.print_filenames { if options.long_listing { - progress_bar.message(format!( - "{} {} {}\n", - entry.unix_mode(), - entry.owner(), - entry.apath() - )); + println!("{} {} {}", entry.unix_mode(), entry.owner(), entry.apath()); } else { - progress_bar.message(format!("{}\n", entry.apath())); + println!("{}", entry.apath()); } } - progress_bar.update(|model| model.filename = entry.apath().to_string()); - if let Err(e) = match entry.kind() { + if !options.print_filenames { + bar.post(Progress::Restore { + filename: entry.apath().to_string(), + bytes_done, + }); + } + if let Err(err) = match entry.kind() { Kind::Dir => { stats.directories += 1; rt.copy_dir(&entry) @@ -131,7 +114,7 @@ pub fn restore( stats.files += 1; let result = rt.copy_file(&entry, &st).map(|s| stats += s); if let Some(bytes) = entry.size() { - progress_bar.update(|model| model.bytes_done += bytes); + bytes_done += bytes; } result } @@ -147,7 +130,10 @@ pub fn restore( continue; } } { - ui::show_error(&e); + error!( + "error restoring {apath}: {err}", + apath = entry.apath().to_string() + ); stats.errors += 1; continue; } @@ -201,13 +187,13 @@ impl RestoreTree { fn finish(self) -> Result { #[cfg(unix)] for (path, unix_mode) in self.dir_unix_modes { - if let Err(err) = unix_mode.set_permissions(path) { - ui::problem(&format!("Failed to set directory permissions: {err:?}")); + if let Err(err) = unix_mode.set_permissions(&path) { + error!("Failed to set directory permissions on {path:?}: {err}"); } } for (path, time) in self.dir_mtimes { - if let Err(err) = filetime::set_file_mtime(path, time.into()) { - ui::problem(&format!("Failed to set directory mtime: {err:?}")); + if let Err(err) = filetime::set_file_mtime(&path, time.into()) { + error!("Failed to set directory mtime on {path:?}: {err}"); } } Ok(RestoreStats::default()) @@ -252,27 +238,19 @@ impl RestoreTree { } })?; - #[cfg(unix)] - { - // Restore permissions only if there are mode bits stored in the archive - source_entry - .unix_mode() - .set_permissions(&path) - .map_err(|e| { - ui::show_error(&e); - stats.errors += 1; - }) - .ok(); + // Restore permissions only if there are mode bits stored in the archive + if let Err(err) = source_entry.unix_mode().set_permissions(&path) { + error!(?path, ?err, "Error restoring unix permissions"); + stats.errors += 1; } // Restore ownership if possible. // TODO: Stats and warnings if a user or group is specified in the index but // does not exist on the local system. - if let Err(err) = source_entry.owner().set_owner(&path) { - ui::show_error(&err); + if let Err(err) = &source_entry.owner().set_owner(&path) { + error!(?path, ?err, "Error restoring ownership"); stats.errors += 1; } - // TODO: Accumulate more stats. Ok(stats) } @@ -291,7 +269,7 @@ impl RestoreTree { } } else { // TODO: Treat as an error. - ui::problem(&format!("No target in symlink entry {}", entry.apath())); + error!("No target in symlink entry {:?}", entry.apath()); } Ok(()) } @@ -300,10 +278,7 @@ impl RestoreTree { fn copy_symlink(&mut self, entry: &E) -> Result<()> { // TODO: Add a test with a canned index containing a symlink, and expect // it cannot be restored on Windows and can be on Unix. - ui::problem(&format!( - "Can't restore symlinks on non-Unix: {}", - entry.apath() - )); + warn!("Can't restore symlinks on non-Unix: {}", entry.apath()); Ok(()) } } diff --git a/src/show.rs b/src/show.rs index 3198a785..844358bc 100644 --- a/src/show.rs +++ b/src/show.rs @@ -22,7 +22,9 @@ use std::io::{BufWriter, Write}; use time::format_description::well_known::Rfc3339; use time::UtcOffset; +use tracing::error; +use crate::misc::duration_to_hms; use crate::*; /// Options controlling the behavior of `show_versions`. @@ -61,15 +63,15 @@ pub fn show_versions( l.push(format!("{band_id:<20}")); let band = match Band::open(archive, &band_id) { Ok(band) => band, - Err(e) => { - ui::problem(&format!("Failed to open band {band_id:?}: {e:?}")); + Err(err) => { + error!("Failed to open band {band_id:?}: {err}"); continue; } }; let info = match band.get_info() { Ok(info) => info, - Err(e) => { - ui::problem(&format!("Failed to read band tail {band_id:?}: {e:?}")); + Err(err) => { + error!("Failed to read band tail {band_id:?}: {err}"); continue; } }; @@ -90,7 +92,7 @@ pub fn show_versions( if let Some(end_time) = info.end_time { let duration = end_time - info.start_time; if let Ok(duration) = duration.try_into() { - crate::ui::duration_to_hms(duration).into() + duration_to_hms(duration).into() } else { Cow::Borrowed("negative") } diff --git a/src/stats.rs b/src/stats.rs index db44b6da..c79ce5cc 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -14,10 +14,10 @@ use std::fmt; use std::time::Duration; -use derive_more::{Add, AddAssign, Sum}; +use derive_more::{Add, AddAssign}; use thousands::Separable; -use crate::ui::duration_to_hms; +use crate::misc::duration_to_hms; pub fn mb_string(s: u64) -> String { (s / 1_000_000).separate_with_commas() @@ -67,69 +67,6 @@ pub struct Sizes { pub uncompressed: u64, } -#[derive(Debug, Default, Clone, PartialEq, Eq, Add, AddAssign, Sum)] -pub struct ValidateStats { - /// Count of files in the wrong place. - pub structure_problems: usize, - pub io_errors: usize, - - /// Failed to open a band. - pub band_open_errors: usize, - pub band_metadata_problems: usize, - pub missing_band_heads: usize, - - /// Failed to open a stored tree. - pub tree_open_errors: usize, - pub tree_validate_errors: usize, - - /// Count of files not expected to be in the archive. - pub unexpected_files: usize, - - /// Number of blocks read. - pub block_read_count: u64, - /// Number of blocks that failed to read back. - pub block_error_count: usize, - pub block_missing_count: usize, - pub block_too_short: usize, - - pub elapsed: Duration, -} - -impl fmt::Display for ValidateStats { - fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.has_problems() { - writeln!(w, "VALIDATION FOUND PROBLEMS")?; - } else { - writeln!(w, "No problems found in archive")?; - } - - write_count(w, "structure problems", self.structure_problems); - write_count(w, "IO errors", self.io_errors); - write_count(w, "band open errors", self.band_open_errors); - write_count(w, "band metadata errors", self.band_metadata_problems); - write_count(w, "missing band heads", self.missing_band_heads); - write_count(w, "tree open errors", self.tree_open_errors); - write_count(w, "tree validate errors", self.tree_validate_errors); - write_count(w, "unexpected files", self.unexpected_files); - writeln!(w).unwrap(); - write_count(w, "block errors", self.block_error_count); - write_count(w, "blocks missing", self.block_too_short); - write_count(w, "blocks too short", self.block_missing_count); - writeln!(w).unwrap(); - - write_count(w, "blocks read", self.block_read_count as usize); - write_duration(w, "elapsed", self.elapsed)?; - - Ok(()) - } -} - -impl ValidateStats { - pub fn has_problems(&self) -> bool { - self.block_error_count > 0 || self.io_errors > 0 || self.block_missing_count > 0 - } -} - #[derive(Default, Debug, Clone, Eq, PartialEq)] pub struct IndexReadStats { pub index_hunks: usize, diff --git a/src/trace_counter.rs b/src/trace_counter.rs new file mode 100644 index 00000000..e881089b --- /dev/null +++ b/src/trace_counter.rs @@ -0,0 +1,41 @@ +// Copyright 2023 Martin Pool. + +//! Count the number of `tracing` errors and warnings. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use tracing::{Event, Level, Subscriber}; +use tracing_subscriber::layer::Context; +use tracing_subscriber::Layer; + +/// Count of errors emitted to trace. +static ERROR_COUNT: AtomicUsize = AtomicUsize::new(0); + +/// Count of warnings emitted to trace. +static WARN_COUNT: AtomicUsize = AtomicUsize::new(0); + +/// Return the number of errors logged in the program so far. +pub fn global_error_count() -> usize { + ERROR_COUNT.load(Ordering::Relaxed) +} + +/// Return the number of warnings logged in the program so far. +pub fn global_warn_count() -> usize { + WARN_COUNT.load(Ordering::Relaxed) +} + +/// A tracing Layer that counts errors and warnings into static counters. +pub(crate) struct CounterLayer(); + +impl Layer for CounterLayer +where + S: Subscriber, +{ + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + match *event.metadata().level() { + Level::ERROR => ERROR_COUNT.fetch_add(1, Ordering::Relaxed), + Level::WARN => WARN_COUNT.fetch_add(1, Ordering::Relaxed), + _ => 0, + }; + } +} diff --git a/src/transport.rs b/src/transport.rs index 3713171c..1f6bb745 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -146,6 +146,9 @@ pub trait Transport: Send + Sync + std::fmt::Debug { /// Return a URL scheme describing this transport, such as "file". fn url_scheme(&self) -> &'static str; + + /// Return a path or URL for this transport. + fn url(&self) -> String; } /// A directory entry read from a transport. diff --git a/src/transport/local.rs b/src/transport/local.rs index 9061c408..2548e916 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -133,6 +133,10 @@ impl Transport for LocalTransport { fn url_scheme(&self) -> &'static str { "file" } + + fn url(&self) -> String { + self.root.to_string_lossy().into() + } } impl AsRef for LocalTransport { diff --git a/src/tree.rs b/src/tree.rs index e8953020..aeb8a2c0 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -15,6 +15,7 @@ use std::ops::Range; +use crate::progress::{Bar, Progress}; use crate::stats::Sizes; use crate::*; @@ -43,38 +44,20 @@ pub trait ReadTree { /// /// This typically requires walking all entries, which may take a while. fn size(&self, exclude: Exclude) -> Result { - struct Model { - files: usize, - total_bytes: u64, - } - impl nutmeg::Model for Model { - fn render(&mut self, _width: usize) -> String { - format!( - "Measuring... {} files, {} MB", - self.files, - self.total_bytes / 1_000_000 - ) - } - } - let progress = nutmeg::View::new( - Model { - files: 0, - total_bytes: 0, - }, - ui::nutmeg_options(), - ); - let mut tot = 0u64; + let mut files = 0; + let mut total_bytes = 0u64; + let bar = Bar::new(); for e in self.iter_entries(Apath::root(), exclude)? { // While just measuring size, ignore directories/files we can't stat. if let Some(bytes) = e.size() { - tot += bytes; - progress.update(|model| { - model.files += 1; - model.total_bytes += bytes; - }); + total_bytes += bytes; + files += 1; + bar.post(Progress::MeasureTree { files, total_bytes }); } } - Ok(TreeSize { file_bytes: tot }) + Ok(TreeSize { + file_bytes: total_bytes, + }) } } diff --git a/src/ui.rs b/src/ui.rs index a9bb4cbb..8e4dc87a 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -11,163 +11,6 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -//! Console UI. +//! Generic UI layer. -use std::fmt::Write; -use std::sync::Mutex; -use std::time::Duration; - -use lazy_static::lazy_static; - -use crate::stats::Sizes; - -/// A terminal/text UI. -/// -/// This manages interleaving log-type messages (info and error), interleaved -/// with progress bars. -/// -/// Progress bars are only drawn when the application requests them with -/// `enable_progress` and the output destination is a tty that's capable -/// of redrawing. -/// -/// So this class also works when stdout is redirected to a file, in -/// which case it will get only messages and no progress bar junk. -#[derive(Default)] -pub(crate) struct UIState { - /// Should a progress bar be drawn? - progress_enabled: bool, -} - -lazy_static! { - static ref UI_STATE: Mutex = Mutex::new(UIState::default()); -} - -pub fn println(s: &str) { - with_locked_ui(|ui| ui.println(s)) -} - -pub fn problem(s: &str) { - with_locked_ui(|ui| ui.problem(s)); -} - -pub(crate) fn with_locked_ui(mut cb: F) -where - F: FnMut(&mut UIState), -{ - use std::ops::DerefMut; - cb(UI_STATE.lock().unwrap().deref_mut()) -} - -pub(crate) fn format_error_causes(error: &dyn std::error::Error) -> String { - let mut buf = error.to_string(); - let mut cause = error; - while let Some(c) = cause.source() { - write!(&mut buf, "\n caused by: {c}").expect("Failed to format error cause"); - cause = c; - } - buf -} - -/// Report that a non-fatal error occurred. -/// -/// The program will continue. -pub fn show_error(e: &dyn std::error::Error) { - // TODO: Log it. - problem(&format_error_causes(e)); -} - -/// Enable drawing progress bars, only if stdout is a tty. -/// -/// Progress bars are off by default. -pub fn enable_progress(enabled: bool) { - let mut ui = UI_STATE.lock().unwrap(); - ui.progress_enabled = enabled; -} - -#[allow(unused)] -pub(crate) fn compression_percent(s: &Sizes) -> i64 { - if s.uncompressed > 0 { - 100i64 - (100 * s.compressed / s.uncompressed) as i64 - } else { - 0 - } -} - -pub(crate) fn duration_to_hms(d: Duration) -> String { - let elapsed_secs = d.as_secs(); - if elapsed_secs >= 3600 { - format!( - "{:2}:{:02}:{:02}", - elapsed_secs / 3600, - (elapsed_secs / 60) % 60, - elapsed_secs % 60 - ) - } else { - format!(" {:2}:{:02}", (elapsed_secs / 60) % 60, elapsed_secs % 60) - } -} - -#[allow(unused)] -pub(crate) fn mbps_rate(bytes: u64, elapsed: Duration) -> f64 { - let secs = elapsed.as_secs() as f64 + f64::from(elapsed.subsec_millis()) / 1000.0; - if secs > 0.0 { - bytes as f64 / secs / 1e6 - } else { - 0f64 - } -} - -/// Describe the compression ratio: higher is better. -#[allow(unused)] -pub(crate) fn compression_ratio(s: &Sizes) -> f64 { - if s.compressed > 0 { - s.uncompressed as f64 / s.compressed as f64 - } else { - 0f64 - } -} - -impl UIState { - pub(crate) fn println(&mut self, s: &str) { - // TODO: Go through Nutmeg instead... - // self.clear_progress(); - println!("{s}"); - } - - fn problem(&mut self, s: &str) { - // TODO: Go through Nutmeg instead... - // self.clear_progress(); - println!("conserve error: {s}"); - // Drawing this way makes messages leak from tests, for unclear reasons. - - // queue!( - // stdout, - // style::SetForegroundColor(style::Color::Red), - // style::SetAttribute(style::Attribute::Bold), - // style::Print("conserve error: "), - // style::SetAttribute(style::Attribute::Reset), - // style::Print(s), - // style::Print("\n"), - // style::ResetColor, - // ) - // .unwrap(); - } -} - -pub(crate) fn nutmeg_options() -> nutmeg::Options { - nutmeg::Options::default().progress_enabled(UI_STATE.lock().unwrap().progress_enabled) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - pub fn test_compression_ratio() { - let ratio = compression_ratio(&Sizes { - compressed: 2000, - uncompressed: 4000, - }); - assert_eq!(format!("{ratio:3.1}x"), "2.0x"); - } -} +pub mod termui; diff --git a/src/ui/termui.rs b/src/ui/termui.rs new file mode 100644 index 00000000..205b6ff1 --- /dev/null +++ b/src/ui/termui.rs @@ -0,0 +1,95 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +//! Terminal/text UI. + +use std::fmt::Debug; +use std::fs::OpenOptions; +use std::path::PathBuf; + +#[allow(unused_imports)] +use tracing::{debug, error, info, trace, warn, Level}; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::fmt::time::FormatTime; +use tracing_subscriber::layer::Layer; +use tracing_subscriber::prelude::*; +use tracing_subscriber::Registry; + +use crate::progress::term::WriteToNutmeg; + +/// Chosen style of timestamp prefix on trace lines. +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum TraceTimeStyle { + /// No timestamp on trace lines. + None, + /// Universal time, in RFC 3339 style. + Utc, + /// Local time, in RFC 3339, using the offset when the program starts. + Local, + /// Time since the start of the process, in seconds. + Relative, +} + +#[must_use] +pub fn enable_tracing( + time_style: &TraceTimeStyle, + console_level: Level, + json_path: &Option, +) -> Option { + use tracing_subscriber::fmt::time; + fn hookup( + timer: FT, + console_level: Level, + json_path: &Option, + ) -> Option + where + FT: FormatTime + Send + Sync + 'static, + { + let console_layer = tracing_subscriber::fmt::Layer::default() + .with_ansi(clicolors_control::colors_enabled()) + .with_writer(WriteToNutmeg) + .with_timer(timer) + .with_filter(LevelFilter::from_level(console_level)); + let json_layer; + let flush_guard; + if let Some(json_path) = json_path { + let file_writer = OpenOptions::new() + .create(true) + .append(true) + .write(true) + .read(false) + .open(json_path) + .expect("open json log file"); + let (non_blocking, guard) = tracing_appender::non_blocking(file_writer); + flush_guard = Some(guard); + json_layer = Some( + tracing_subscriber::fmt::Layer::default() + .json() + .with_writer(non_blocking), + ); + } else { + flush_guard = None; + json_layer = None; + } + Registry::default() + .with(console_layer) + .with(crate::trace_counter::CounterLayer()) + .with(json_layer) + .init(); + flush_guard + } + + let flush_guard = match time_style { + TraceTimeStyle::None => hookup((), console_level, json_path), + TraceTimeStyle::Utc => hookup(time::UtcTime::rfc_3339(), console_level, json_path), + TraceTimeStyle::Relative => hookup(time::uptime(), console_level, json_path), + TraceTimeStyle::Local => hookup( + time::OffsetTime::local_rfc_3339().unwrap(), + console_level, + json_path, + ), + }; + trace!("Tracing enabled"); + flush_guard +} diff --git a/src/unix_mode.rs b/src/unix_mode.rs index 9b2bc062..dc6c409e 100644 --- a/src/unix_mode.rs +++ b/src/unix_mode.rs @@ -1,6 +1,6 @@ -// Copyright 2022 Stephanie Aelmore. // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2022 Stephanie Aelmore. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -31,13 +31,13 @@ //! using the write bit in the user class. //! TODO: Properly implement and test Windows compatibility. //! + +use std::fmt; +use std::fs::Permissions; +use std::io; +use std::path::Path; + use serde::{Deserialize, Serialize}; -use std::{ - fmt, - fs::{self, Permissions}, - io, - path::Path, -}; use unix_mode; #[derive(Debug, Default, Clone, Copy, PartialOrd, Ord, Serialize, Deserialize)] @@ -57,7 +57,7 @@ impl UnixMode { { if let Some(mode) = self.0 { let permissions = Permissions::from_mode(mode); - fs::set_permissions(&path, permissions) + std::fs::set_permissions(&path, permissions) } else { Ok(()) } diff --git a/src/validate.rs b/src/validate.rs index dd0a670f..b1121d7a 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -12,83 +12,80 @@ use std::cmp::max; use std::collections::HashMap; +use std::fmt::Debug; use std::time::Instant; +#[allow(unused_imports)] +use tracing::{error, info, warn}; + +use crate::misc::ResultExt; +use crate::progress::{Bar, Progress}; use crate::*; +/// Options to [Archive::validate]. #[derive(Debug, Default)] pub struct ValidateOptions { /// Assume blocks that are present have the right content: don't read and hash them. pub skip_block_hashes: bool, } +/// Validate the indexes of all bands. +/// +/// Returns the lengths of all blocks that were referenced, so that the caller can check +/// that all blocks are present and long enough. pub(crate) fn validate_bands( archive: &Archive, band_ids: &[BandId], -) -> (HashMap, ValidateStats) { - let mut stats = ValidateStats::default(); +) -> Result> { let mut block_lens = HashMap::new(); - struct ProgressModel { - bands_done: usize, - bands_total: usize, - start: Instant, - } - impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Check index {}/{}, {} done, {} remaining", - self.bands_done, - self.bands_total, - nutmeg::percent_done(self.bands_done, self.bands_total), - nutmeg::estimate_remaining(&self.start, self.bands_done, self.bands_total) - ) - } - } - let view = nutmeg::View::new( - ProgressModel { - start: Instant::now(), - bands_done: 0, - bands_total: band_ids.len(), - }, - ui::nutmeg_options(), - ); - for band_id in band_ids { - if let Ok(b) = Band::open(archive, band_id) { - if b.validate(&mut stats).is_err() { - stats.band_metadata_problems += 1; + let start = Instant::now(); + let total_bands = band_ids.len(); + let bar = Bar::new(); + 'band: for (bands_done, band_id) in band_ids.iter().enumerate() { + let band = match Band::open(archive, band_id) { + Ok(band) => band, + Err(err) => { + error!(%err, %band_id, "Error opening band"); + continue 'band; } - } else { - stats.band_open_errors += 1; - continue; + }; + if let Err(err) = band.validate() { + error!(%err, %band_id, "Error validating band"); + continue 'band; + }; + if let Err(err) = archive + .open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) + .and_then(|st| validate_stored_tree(&st)) + .map(|st_block_lens| merge_block_lens(&mut block_lens, &st_block_lens)) + { + error!(%err, %band_id, "Error validating stored tree"); + continue 'band; } - if let Ok(st) = archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { - if let Ok((st_block_lens, st_stats)) = validate_stored_tree(&st) { - stats += st_stats; - for (bh, bl) in st_block_lens { - block_lens - .entry(bh) - .and_modify(|al| *al = max(*al, bl)) - .or_insert(bl); - } - } else { - stats.tree_validate_errors += 1 - } - } else { - stats.tree_open_errors += 1; - continue; - } - view.update(|model| model.bands_done += 1); + bar.post(Progress::ValidateBands { + total_bands, + bands_done, + start, + }); + } + Ok(block_lens) +} + +fn merge_block_lens(into: &mut HashMap, from: &HashMap) { + for (bh, bl) in from { + into.entry(bh.clone()) + .and_modify(|l| *l = max(*l, *bl)) + .or_insert(*bl); } - (block_lens, stats) } -pub(crate) fn validate_stored_tree( - st: &StoredTree, -) -> Result<(HashMap, ValidateStats)> { +fn validate_stored_tree(st: &StoredTree) -> Result> { let mut block_lens = HashMap::new(); - let stats = ValidateStats::default(); + // TODO: Check other entry properties are correct. + // TODO: Check they're in apath order. + // TODO: Count progress for index blocks within one tree? for entry in st - .iter_entries(Apath::root(), Exclude::nothing())? + .iter_entries(Apath::root(), Exclude::nothing()) + .our_inspect_err(|err| error!(%err, "Error iterating index entries"))? .filter(|entry| entry.kind() == Kind::File) { for addr in entry.addrs { @@ -99,5 +96,5 @@ pub(crate) fn validate_stored_tree( .or_insert(end); } } - Ok((block_lens, stats)) + Ok(block_lens) } diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 56f526de..5d954c44 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -1,4 +1,4 @@ -// Copyright 2015, 2016, 2017, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -20,6 +20,7 @@ use conserve::kind::Kind; use conserve::test_fixtures::ScratchArchive; use conserve::test_fixtures::TreeFixture; use conserve::*; +use tracing_test::traced_test; const HELLO_HASH: &str = "9063990e5c5b2184877f92adace7c801a549b00c39cd7549877f06d5dd0d3a6ca6eee42d5\ @@ -53,6 +54,7 @@ pub fn simple_backup() { } #[test] +#[traced_test] pub fn simple_backup_with_excludes() -> Result<()> { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); @@ -98,8 +100,8 @@ pub fn simple_backup_with_excludes() -> Result<()> { // TODO: Check index stats. // TODO: Check what was restored. - let validate_stats = af.validate(&ValidateOptions::default()).unwrap(); - assert!(!validate_stats.has_problems()); + af.validate(&ValidateOptions::default()).unwrap(); + assert!(!logs_contain("ERROR") && !logs_contain("WARN")); Ok(()) } diff --git a/tests/api/damaged.rs b/tests/api/damaged.rs index a42595a2..726d602a 100644 --- a/tests/api/damaged.rs +++ b/tests/api/damaged.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2020, Martin Pool. +// Copyright 2020-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -15,26 +15,28 @@ use std::path::Path; +use tracing_test::traced_test; + use conserve::*; +#[traced_test] #[test] -fn missing_block() -> Result<()> { +fn missing_block_when_checking_hashes() -> Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; - - let validate_stats = archive.validate(&ValidateOptions::default())?; - assert!(validate_stats.has_problems()); - assert_eq!(validate_stats.block_missing_count, 1); + archive.validate(&ValidateOptions::default())?; + assert!(logs_contain( + "Referenced block missing block_hash=fec91c70284c72d0d4e3684788a90de9338a5b2f47f01fedbe203cafd68708718ae5672d10eca804a8121904047d40d1d6cf11e7a76419357a9469af41f22d01")); Ok(()) } +#[traced_test] #[test] fn missing_block_skip_block_hashes() -> Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; - - let validate_stats = archive.validate(&ValidateOptions { + archive.validate(&ValidateOptions { skip_block_hashes: true, })?; - assert!(validate_stats.has_problems()); - assert_eq!(validate_stats.block_missing_count, 1); + assert!(logs_contain( + "Referenced block missing block_hash=fec91c70284c72d0d4e3684788a90de9338a5b2f47f01fedbe203cafd68708718ae5672d10eca804a8121904047d40d1d6cf11e7a76419357a9469af41f22d01")); Ok(()) } diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index f10e58e3..e1c4030d 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -24,6 +24,7 @@ use pretty_assertions::assert_eq; use conserve::unix_time::UnixTime; use conserve::*; +use tracing_test::traced_test; const MINIMAL_ARCHIVE_VERSIONS: &[&str] = &["0.6.0", "0.6.10", "0.6.2", "0.6.3", "0.6.9", "0.6.17"]; @@ -71,19 +72,17 @@ fn examine_archive() { } } +#[traced_test] #[test] fn validate_archive() { for ver in MINIMAL_ARCHIVE_VERSIONS { println!("validate {ver}"); let archive = open_old_archive(ver, "minimal"); - let stats = archive + archive .validate(&ValidateOptions::default()) .expect("validate archive"); - assert_eq!(stats.structure_problems, 0); - assert_eq!(stats.io_errors, 0); - assert_eq!(stats.block_error_count, 0); - assert!(!stats.has_problems()); + assert!(!logs_contain("ERROR") && !logs_contain("WARN")); } } diff --git a/tests/cli/delete.rs b/tests/cli/delete.rs index f8a5fef8..a9a6fc83 100644 --- a/tests/cli/delete.rs +++ b/tests/cli/delete.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2016, 2017, 2018, 2019, 2020 Martin Pool. +// Copyright 2016-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -118,18 +118,17 @@ fn delete_second_version() { fn delete_nonexistent_band() { let af = ScratchArchive::new(); - let pred_fn = predicate::str::is_match( - r"conserve error: Failed to delete band b0000 - caused by: (No such file or directory|The system cannot find the file specified\.) \(os error \d+\) -", - ) - .unwrap(); - run_conserve() .args(["delete"]) .args(["-b", "b0000"]) .arg(af.path()) .assert() - .stdout(pred_fn) + .stderr(predicate::str::contains( + "ERROR conserve: Failed to delete band b0000", + )) + .stderr( + predicate::str::is_match(r#"caused by: (File not found.|No such file or directory|The system cannot find the file specified\.) \(os error \d+\)"#) + .unwrap(), + ) .failure(); } diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 4c84f1eb..801ec1ff 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2016-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ mod backup; mod delete; mod diff; mod exclude; +mod trace; mod validate; mod versions; @@ -74,7 +75,7 @@ fn clean_error_on_non_archive() { .arg(".") .assert() .failure() - .stdout(predicate::str::contains("Not a Conserve archive")); + .stderr(predicate::str::contains("Not a Conserve archive")); } #[test] @@ -89,7 +90,7 @@ fn basic_backup() { .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with("Created new archive")); + .stdout(predicate::str::is_empty()); // New archive contains no versions. run_conserve() @@ -131,8 +132,8 @@ fn basic_backup() { .arg(&src) .assert() .success() - .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with("Backup complete.\n")); + .stderr(predicate::str::contains("Backup complete.")) + .stdout(predicate::str::is_empty()); // TODO: Now inspect the archive. run_conserve() @@ -230,7 +231,7 @@ fn basic_backup() { /hello\n\ /subdir\n\ /subdir/subfile\n\ - Restore complete.\n", + ", )); restore_dir @@ -253,7 +254,7 @@ fn basic_backup() { .arg(restore_dir.path()) .assert() .failure() - .stdout(predicate::str::contains("Destination directory not empty")); + .stderr(predicate::str::contains("Destination directory not empty")); // Restore with specified band id / backup version. { @@ -274,8 +275,8 @@ fn basic_backup() { .arg(arch_dir) .assert() .success() - .stderr(predicate::str::is_empty()) - .stdout(predicate::str::contains("Archive is OK.\n")); + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("Archive is OK.\n")); // TODO: Compare vs source tree. } @@ -294,21 +295,22 @@ fn empty_archive() { .arg(restore_dir.path()) .assert() .failure() - .stdout(predicate::str::contains("Archive has no bands")); + .stderr(predicate::str::contains("Archive has no bands")); run_conserve() .arg("ls") .arg(&adir) .assert() .failure() - .stdout(predicate::str::contains("Archive has no bands")); + .stderr(predicate::str::contains("Archive has no bands")); run_conserve() .arg("versions") .arg(&adir) .assert() .success() - .stdout(predicate::str::is_empty()); + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::is_empty()); run_conserve().arg("gc").arg(adir).assert().success(); } @@ -339,16 +341,7 @@ fn incomplete_version() { .arg(af.path()) .assert() .failure() - .stdout(predicate::str::contains("incomplete and may be in use")); -} - -#[test] -fn validate_non_fatal_problems_nonzero_result() { - run_conserve() - .args(["validate", "testdata/damaged/missing-block/"]) - .assert() - .stdout(predicate::str::contains("Archive has some problems.")) - .code(2); + .stderr(predicate::str::contains("incomplete and may be in use")); } #[test] diff --git a/tests/cli/trace.rs b/tests/cli/trace.rs new file mode 100644 index 00000000..e5dc8c12 --- /dev/null +++ b/tests/cli/trace.rs @@ -0,0 +1,21 @@ +// Copyright 2023 Martin Pool + +//! Tests for trace-related options and behaviors of the Conserve CLI. + +use assert_fs::prelude::*; +use predicates::prelude::*; + +use super::*; + +#[test] +fn no_trace_timestamps_by_default() { + let temp_dir = TempDir::new().unwrap(); + run_conserve() + .args(["-D", "init"]) + .arg(temp_dir.child("archive").path()) + .assert() + .success() + .stderr(predicate::str::contains( + "TRACE conserve::ui::termui: Tracing enabled", + )); +} diff --git a/tests/cli/unix/permissions.rs b/tests/cli/unix/permissions.rs index 4e38e02b..0efe5e30 100644 --- a/tests/cli/unix/permissions.rs +++ b/tests/cli/unix/permissions.rs @@ -29,8 +29,7 @@ fn backup_unix_permissions() { .arg(&arch_dir) .assert() .success() - .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with("Created new archive")); + .stderr(predicate::str::is_empty()); // copy the appropriate testdata into the testdir let src: PathBuf = "./testdata/tree/minimal".into(); @@ -88,7 +87,7 @@ fn backup_unix_permissions() { .arg(&data_dir) .assert() .success() - .stderr(predicate::str::is_empty()) + .stderr(predicate::str::contains("Backup complete.")) .stdout(predicate::str::starts_with(expected)); // verify file permissions in stored archive @@ -116,12 +115,12 @@ fn backup_unix_permissions() { .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with(format!( + .stdout(predicate::str::diff(format!( "rwxr-xr-x {user:<10} {group:<10} /\n\ r--r--r-- {user:<10} {group:<10} /hello\n\ rwxrwxr-x {user:<10} {group:<10} /subdir\n\ rwxr-xr-x {user:<10} {group:<10} /subdir/subfile\n\ - Restore complete.\n" + " ))); } @@ -138,8 +137,7 @@ fn backup_user_and_permissions() { .arg(&arch_dir) .assert() .success() - .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with("Created new archive")); + .stderr(predicate::str::is_empty()); let src: PathBuf = "./testdata/tree/minimal".into(); assert!(src.is_dir()); @@ -197,8 +195,8 @@ fn backup_user_and_permissions() { .arg(&src) .assert() .success() - .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with("Backup complete.\n")); + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("Backup complete.\n")); let restore_dir = TempDir::new().unwrap(); @@ -215,7 +213,7 @@ fn backup_user_and_permissions() { {} {} /hello\n\ {} {} /subdir\n\ {} {} /subdir/subfile\n\ - Restore complete.\n", + ", UnixMode::from(mdata_root.permissions()), Owner::from(&mdata_root), UnixMode::from(mdata_hello.permissions()), diff --git a/tests/cli/validate.rs b/tests/cli/validate.rs index d99ec3a2..a88887a2 100644 --- a/tests/cli/validate.rs +++ b/tests/cli/validate.rs @@ -1,16 +1,50 @@ +// Copyright 2023 Martin Pool + //! Tests for the `conserve validate` CLI. +use std::path::Path; + use assert_cmd::prelude::*; use assert_fs::prelude::*; -use assert_fs::TempDir; +use assert_fs::{NamedTempFile, TempDir}; use predicates::prelude::*; +use serde_json::json; +use serde_json::{Deserializer, Value}; +use tracing::Level; use super::run_conserve; +fn read_log_json(path: &Path) -> Vec { + let json_content = std::fs::read_to_string(path).unwrap(); + println!("{json_content}"); + Deserializer::from_str(&json_content) + .into_iter::() + .map(Result::unwrap) + .collect::>() +} + +/// Filter out only logs with severity equal or more important than `level`. +fn filter_by_level(logs: &[serde_json::Value], level: Level) -> Vec<&serde_json::Value> { + logs.iter() + .filter(move |event| event["level"].as_str().unwrap().parse::().unwrap() <= level) + .collect() +} + +// /// Reduce json logs to just their messages. +// fn events_to_messages<'s, I>(logs: I) -> Vec<&'s str> +// where +// I: IntoIterator, +// { +// logs.into_iter() +// .map(|event| event["fields"]["message"].as_str().unwrap()) +// .collect() +// } + /// #[test] fn validate_does_not_complain_about_gc_lock() { let temp = TempDir::new().unwrap(); + let log_temp = NamedTempFile::new("log.json").unwrap(); run_conserve() .args(["init"]) .arg(temp.path()) @@ -19,8 +53,36 @@ fn validate_does_not_complain_about_gc_lock() { temp.child("GC_LOCK").touch().unwrap(); run_conserve() .args(["validate"]) + .arg("--log-json") + .arg(log_temp.path()) .arg(temp.path()) .assert() .stdout(predicate::str::contains("Unexpected file").not()) .success(); + let events = read_log_json(log_temp.path()); + dbg!(&events); + assert!(filter_by_level(&events, Level::WARN).is_empty()); +} + +#[test] +fn validate_non_fatal_problems_nonzero_result_and_json_log() { + let log_temp = NamedTempFile::new("log.json").unwrap(); + run_conserve() + .args(["validate", "testdata/damaged/missing-block/"]) + .arg("--log-json") + .arg(log_temp.path()) + .assert() + .stderr(predicate::str::contains("Archive has some problems.")) + .code(2); + let events = read_log_json(log_temp.path()); + dbg!(&events); + let errors = filter_by_level(&events, Level::ERROR); + assert_eq!(errors.len(), 1); + assert_eq!( + errors[0]["fields"], + json!({ + "block_hash": "fec91c70284c72d0d4e3684788a90de9338a5b2f47f01fedbe203cafd68708718ae5672d10eca804a8121904047d40d1d6cf11e7a76419357a9469af41f22d01", + "message": "Referenced block missing", + }) + ); } From a2f6d60f824e1a66af2bc856c962e7e0368ae0a7 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 06:06:42 -0800 Subject: [PATCH 02/86] Bump band version to 23.2 --- src/band.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/band.rs b/src/band.rs index 47fe1cfb..5d2db79f 100644 --- a/src/band.rs +++ b/src/band.rs @@ -36,7 +36,7 @@ static INDEX_DIR: &str = "i"; /// Band format-compatibility. Bands written out by this program, can only be /// read correctly by versions equal or later than the stated version. -pub const BAND_FORMAT_VERSION: &str = "0.6.3"; +pub const BAND_FORMAT_VERSION: &str = "23.2.0"; /// Describes how to select a band from an archive. #[derive(Debug, Clone, Eq, PartialEq)] @@ -50,13 +50,13 @@ pub enum BandSelectionPolicy { } fn band_version_requirement() -> semver::VersionReq { - semver::VersionReq::parse("<=0.6.3").unwrap() + semver::VersionReq::parse(&format!("<={BAND_FORMAT_VERSION}")).unwrap() } fn band_version_supported(version: &str) -> bool { semver::Version::parse(version) .map(|sv| band_version_requirement().matches(&sv)) - .unwrap_or(false) + .unwrap() } /// Each backup makes a new `band` containing an index directory. @@ -300,7 +300,7 @@ mod tests { fs::create_dir(af.path().join("b0000")).unwrap(); let head = json!({ "start_time": 0, - "band_format_version": "0.8.8", + "band_format_version": "8888.8.8", }); fs::write( af.path().join("b0000").join(BAND_HEAD_FILENAME), @@ -311,7 +311,7 @@ mod tests { let e = Band::open(&af, &BandId::zero()); let e_str = e.unwrap_err().to_string(); assert!( - e_str.contains("Band version \"0.8.8\" in"), + e_str.contains("Band version \"8888.8.8\" in"), "bad band version: {e_str:#?}" ); } From 6ff4d7dd8e1ed4e425751c387cea26d7a8a5e145 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 07:05:39 -0800 Subject: [PATCH 03/86] Add per-band format flags --- doc/format.md | 11 ++++---- doc/unimplemented/format7.md | 18 ++++++++---- src/band.rs | 53 +++++++++++++++++++++++++++++++++--- src/errors.rs | 11 ++++++++ tests/api/format_flags.rs | 24 ++++++++++++++++ tests/api/main.rs | 3 +- 6 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 tests/api/format_flags.rs diff --git a/doc/format.md b/doc/format.md index 7dac4b00..969bcaba 100644 --- a/doc/format.md +++ b/doc/format.md @@ -119,11 +119,6 @@ In Conserve 0.6, only bands with a single integer, called a _top level band_, are generated or supported. Top level bands contain an index listing every entry present in that tree. Top level bands are numbered sequentially from `b0000`. -Bands that are not top-level are _child bands_, and their _parent band_ is the -band with the last component of their name removed. Child bands' index contains -only the changes relative to their parent band's index. (Child bands are not -implemented as of Conserve 0.6.) - A band can be _complete_, while it is receiving data, or _incomplete_ when everything from the source has been written. Bands may remain incomplete indefinitely, across multiple Conserve invocations, until they are finished. @@ -158,6 +153,8 @@ The head file contains: - `start_time`: The Unix time, in seconds, when the band was started. - `band_format_version`: The minimum program version to correctly read this band. +- `format_flags`: A list of strings indicating capabilities required to read + this band correctly. Only present from 23.2 onwards. ### Band tail file @@ -170,6 +167,10 @@ Band footer contains: - `index_hunk_count`: The number of index hunks that should be present for this band. (Since 0.6.4.) +## Format flags + +(None are defined yet.) + ## Data block directory An archive contains a single data block directory, which stores the compressed diff --git a/doc/unimplemented/format7.md b/doc/unimplemented/format7.md index c2ac1c7f..a3cf83e2 100644 --- a/doc/unimplemented/format7.md +++ b/doc/unimplemented/format7.md @@ -20,11 +20,19 @@ Band ids currently support a dashed-decimal syntax and are internally a `Vec] = &[]; + + /// All the flags understood by this version of Conserve. + pub static SUPPORTED: &[Cow<'static, str>] = &[]; +} + /// Describes how to select a band from an archive. #[derive(Debug, Clone, Eq, PartialEq)] pub enum BandSelectionPolicy { @@ -79,6 +91,11 @@ struct Head { /// Semver string for the minimum Conserve version to read this band /// correctly. band_format_version: Option, + + /// Format flags that must be understood to read this band and the + /// referenced data correctly. + #[serde(default)] + format_flags: Vec>, } /// Format of the on-disk tail file. @@ -115,6 +132,13 @@ impl Band { /// /// The Band gets the next id after those that already exist. pub fn create(archive: &Archive) -> Result { + Band::create_with_flags(archive, flags::DEFAULT) + } + + pub fn create_with_flags( + archive: &Archive, + format_flags: &[Cow<'static, str>], + ) -> Result { let band_id = archive .last_band_id()? .map_or_else(BandId::zero, |b| b.next_sibling()); @@ -126,6 +150,7 @@ impl Band { let head = Head { start_time: OffsetDateTime::now_utc().unix_timestamp(), band_format_version: Some(BAND_FORMAT_VERSION.to_owned()), + format_flags: format_flags.into(), }; write_json(&transport, BAND_HEAD_FILENAME, &head)?; Ok(Band { @@ -159,9 +184,24 @@ impl Band { }); } } else { + debug!("Old(?) band {band_id} has no format version"); // Unmarked, old bands, are accepted for now. In the next archive // version, band version markers ought to become mandatory. } + + let unsupported_flags = head + .format_flags + .iter() + .filter(|f| !flags::SUPPORTED.contains(f)) + .cloned() + .collect_vec(); + if !unsupported_flags.is_empty() { + return Err(Error::UnsupportedBandFormatFlags { + band_id: band_id.clone(), + unsupported_flags, + }); + } + Ok(Band { band_id: band_id.to_owned(), head, @@ -191,6 +231,11 @@ impl Band { &self.band_id } + /// Get the format flags in this band, from [flags]. + pub fn format_flags(&self) -> &[Cow<'static, str>] { + &self.head.format_flags + } + pub fn index_builder(&self) -> IndexWriter { IndexWriter::new(self.transport.sub_transport(INDEX_DIR)) } diff --git a/src/errors.rs b/src/errors.rs index 0ca9db7c..e34ede08 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -13,6 +13,7 @@ //! Conserve error types. +use std::borrow::Cow; use std::io; use std::path::PathBuf; @@ -107,6 +108,16 @@ pub enum Error { )] UnsupportedBandVersion { band_id: BandId, version: String }, + #[error( + "Band {band_id} has feature flags {unsupported_flags:?} \ + not supported by Conserve {conserve_version}", + conserve_version = crate::version() + )] + UnsupportedBandFormatFlags { + band_id: BandId, + unsupported_flags: Vec>, + }, + #[error("Destination directory not empty: {:?}", path)] DestinationNotEmpty { path: PathBuf }, diff --git a/tests/api/format_flags.rs b/tests/api/format_flags.rs new file mode 100644 index 00000000..4fd296db --- /dev/null +++ b/tests/api/format_flags.rs @@ -0,0 +1,24 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +//! Tests for per-band format flags. + +use assert_matches::assert_matches; + +use conserve::test_fixtures::ScratchArchive; +use conserve::*; + +#[test] +fn unknown_format_flag_fails_to_open() { + let af = ScratchArchive::new(); + + let orig_band = Band::create_with_flags(&af, &["wibble".into()]).unwrap(); + assert_eq!(orig_band.format_flags(), ["wibble"]); + + let err = Band::open(&af, orig_band.id()).unwrap_err(); + println!("{err}"); + assert_matches!(err, Error::UnsupportedBandFormatFlags { .. }); + assert!(err + .to_string() + .starts_with(r#"Band b0000 has feature flags ["wibble"] not supported by Conserve "#)); +} diff --git a/tests/api/main.rs b/tests/api/main.rs index 92cddbd1..cb6032fc 100644 --- a/tests/api/main.rs +++ b/tests/api/main.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Martin Pool. +// Copyright 2021-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ mod blockhash; mod damaged; mod delete; mod diff; +mod format_flags; mod gc; mod live_tree; mod old_archives; From 79f342b11c4a2a9a17f420492720cfffd95a1c56 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 08:20:10 -0800 Subject: [PATCH 04/86] Bump version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 684a0537..6694ae5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,7 +216,7 @@ dependencies = [ [[package]] name = "conserve" -version = "23.1.1" +version = "23.2.0-pre" dependencies = [ "assert_cmd", "assert_fs", diff --git a/Cargo.toml b/Cargo.toml index 17b417d5..a680ccc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-2.0" name = "conserve" readme = "README.md" repository = "https://github.com/sourcefrog/conserve/" -version = "23.1.1" +version = "23.2.0-pre" rust-version = "1.63" [[bin]] From 13e86facba6f044569c691deaa5fd3e4fc6162e2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 08:18:11 -0800 Subject: [PATCH 05/86] Remove dashed band ids This was supported in the type for a long time, but never actually generated. It no longer seems like a good idea: rather than bands having explicit parents, they can just share blocks. --- src/bandid.rs | 114 ++++++++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 65 deletions(-) diff --git a/src/bandid.rs b/src/bandid.rs index 621ffc96..fe385974 100644 --- a/src/bandid.rs +++ b/src/bandid.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -11,45 +11,36 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -//! Bands are identified by a string like `b0001-0023`, represented by a `BandId` object. +//! Bands are identified by a string like `b0001`, represented by a [BandId] object. -use std::fmt::{self, Write}; +use std::fmt; use std::str::FromStr; use serde::Serialize; use crate::errors::Error; -/// Identifier for a band within an archive, eg 'b0001' or 'b0001-0020'. -/// -/// `BandId`s implement a total ordering `std::cmp::Ord`. +/// Identifier for a band within an archive, eg 'b0001'. #[derive(Debug, PartialEq, Clone, Eq, PartialOrd, Ord, Hash, Serialize)] -pub struct BandId { - /// The sequence numbers at each tier. - seqs: Vec, -} +pub struct BandId(u32); impl BandId { /// Makes a new BandId from a sequence of integers. pub fn new(seqs: &[u32]) -> BandId { - assert!(!seqs.is_empty()); - BandId { - seqs: seqs.to_vec(), - } + assert_eq!(seqs.len(), 1, "Band id should have a single element"); + BandId(seqs[0]) } /// Return the origin BandId. #[must_use] pub fn zero() -> BandId { - BandId::new(&[0]) + BandId(0) } /// Return the next BandId at the same level as self. #[must_use] pub fn next_sibling(&self) -> BandId { - let mut next_seqs = self.seqs.clone(); - next_seqs[self.seqs.len() - 1] += 1; - BandId::new(&next_seqs) + BandId(self.0 + 1) } /// Return the previous band, unless this is zero. @@ -59,13 +50,10 @@ impl BandId { /// Currently only implemented for top-level bands. #[must_use] pub fn previous(&self) -> Option { - if self.seqs.len() != 1 { - unimplemented!("BandId::previous only supported on len 1") - } - if self.seqs[0] == 0 { + if self.0 == 0 { None } else { - Some(BandId::new(&[self.seqs[0] - 1])) + Some(BandId(self.0 - 1)) } } } @@ -75,22 +63,18 @@ impl FromStr for BandId { /// Make a new BandId from a string form. fn from_str(s: &str) -> std::result::Result { - let nope = || Err(Error::InvalidVersion { version: s.into() }); - if !s.starts_with('b') { - return nope(); - } - let mut seqs = Vec::::new(); - for num_part in s[1..].split('-') { - match num_part.parse::() { - Ok(num) => seqs.push(num), - Err(..) => return nope(), + if let Some(num) = s.strip_prefix('b') { + if let Ok(num) = num.parse::() { + return Ok(BandId(num)); } } - if seqs.is_empty() { - nope() - } else { - Ok(BandId::new(&seqs)) - } + Err(Error::InvalidVersion { version: s.into() }) + } +} + +impl From for BandId { + fn from(value: u32) -> Self { + BandId(value) } } @@ -104,19 +88,14 @@ impl fmt::Display for BandId { /// Numbers are zero-padded to what should normally be a reasonable length, /// but they can be longer. fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut result = String::with_capacity(self.seqs.len() * 5); - result.push('b'); - for s in &self.seqs { - let _ = write!(result, "{s:04}-"); - } - result.pop(); // remove the last dash - result.shrink_to_fit(); - f.pad(&result) + f.pad(&format!("b{:0>4}", self.0)) } } #[cfg(test)] mod tests { + use assert_matches::assert_matches; + use super::*; #[test] @@ -149,22 +128,24 @@ mod tests { } #[test] - fn next() { + fn next_of_zero_is_one() { assert_eq!(BandId::zero().next_sibling().to_string(), "b0001"); - assert_eq!( - BandId::new(&[2, 3]).next_sibling().to_string(), - "b0002-0004" - ); + } + + #[test] + fn next_of_two_is_three() { + assert_eq!(BandId::from(2).next_sibling().to_string(), "b0003"); } #[test] fn to_string() { - let band_id = BandId::new(&[1, 10, 20]); - assert_eq!(band_id.to_string(), "b0001-0010-0020"); - assert_eq!( - BandId::new(&[1_000_000, 2_000_000]).to_string(), - "b1000000-2000000" - ) + let band_id = BandId::new(&[20]); + assert_eq!(band_id.to_string(), "b0020"); + } + + #[test] + fn large_value_to_string() { + assert_eq!(BandId::new(&[2_000_000]).to_string(), "b2000000") } #[test] @@ -187,17 +168,20 @@ mod tests { fn from_string_valid() { assert_eq!(BandId::from_str("b0001").unwrap().to_string(), "b0001"); assert_eq!(BandId::from_str("b123456").unwrap().to_string(), "b123456"); - assert_eq!( - BandId::from_str("b0001-0100-0234").unwrap().to_string(), - "b0001-0100-0234" - ); } #[test] - fn format() { - let a_bandid = BandId::from_str("b0001-0234").unwrap(); - assert_eq!(format!("{a_bandid}"), "b0001-0234"); - // Implements padding correctly - assert_eq!(format!("{a_bandid:<15}"), "b0001-0234 "); + fn dashes_are_no_longer_valid() { + // Versions prior to 23.2 accepted bandids with dashes, but never + // used them. + let err = BandId::from_str("b0001-0100-0234").unwrap_err(); + assert_matches!(err, Error::InvalidVersion { .. }); + } + + #[test] + fn to_string_respects_padding() { + let s = format!("{:<10}", BandId::from(42)); + assert_eq!(s.len(), 10); + assert_eq!(s, "b0042 "); } } From d83d292f6e4569f8f764bdf67a9a0ff80f4d45b4 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 08:25:40 -0800 Subject: [PATCH 06/86] Move BandId tests to integration tests --- src/bandid.rs | 94 -------------------------------------------- tests/api/bandid.rs | 95 +++++++++++++++++++++++++++++++++++++++++++++ tests/api/main.rs | 1 + 3 files changed, 96 insertions(+), 94 deletions(-) create mode 100644 tests/api/bandid.rs diff --git a/src/bandid.rs b/src/bandid.rs index fe385974..5d9b9e27 100644 --- a/src/bandid.rs +++ b/src/bandid.rs @@ -91,97 +91,3 @@ impl fmt::Display for BandId { f.pad(&format!("b{:0>4}", self.0)) } } - -#[cfg(test)] -mod tests { - use assert_matches::assert_matches; - - use super::*; - - #[test] - #[should_panic] - fn empty_id_not_allowed() { - BandId::new(&[]); - } - - #[test] - fn equality() { - assert_eq!(BandId::new(&[1]), BandId::new(&[1])) - } - - #[test] - fn zero() { - assert_eq!(BandId::zero().to_string(), "b0000"); - } - - #[test] - fn zero_has_no_previous() { - assert_eq!(BandId::zero().previous(), None); - } - - #[test] - fn previous_of_one_is_zero() { - assert_eq!( - BandId::zero().next_sibling().previous(), - Some(BandId::zero()) - ); - } - - #[test] - fn next_of_zero_is_one() { - assert_eq!(BandId::zero().next_sibling().to_string(), "b0001"); - } - - #[test] - fn next_of_two_is_three() { - assert_eq!(BandId::from(2).next_sibling().to_string(), "b0003"); - } - - #[test] - fn to_string() { - let band_id = BandId::new(&[20]); - assert_eq!(band_id.to_string(), "b0020"); - } - - #[test] - fn large_value_to_string() { - assert_eq!(BandId::new(&[2_000_000]).to_string(), "b2000000") - } - - #[test] - fn from_string_detects_invalid() { - assert!(BandId::from_str("").is_err()); - assert!(BandId::from_str("hello").is_err()); - assert!(BandId::from_str("b").is_err()); - assert!(BandId::from_str("b-").is_err()); - assert!(BandId::from_str("b2-").is_err()); - assert!(BandId::from_str("b-2").is_err()); - assert!(BandId::from_str("b2-1-").is_err()); - assert!(BandId::from_str("b2--1").is_err()); - assert!(BandId::from_str("beta").is_err()); - assert!(BandId::from_str("b-eta").is_err()); - assert!(BandId::from_str("b-1eta").is_err()); - assert!(BandId::from_str("b-1-eta").is_err()); - } - - #[test] - fn from_string_valid() { - assert_eq!(BandId::from_str("b0001").unwrap().to_string(), "b0001"); - assert_eq!(BandId::from_str("b123456").unwrap().to_string(), "b123456"); - } - - #[test] - fn dashes_are_no_longer_valid() { - // Versions prior to 23.2 accepted bandids with dashes, but never - // used them. - let err = BandId::from_str("b0001-0100-0234").unwrap_err(); - assert_matches!(err, Error::InvalidVersion { .. }); - } - - #[test] - fn to_string_respects_padding() { - let s = format!("{:<10}", BandId::from(42)); - assert_eq!(s.len(), 10); - assert_eq!(s, "b0042 "); - } -} diff --git a/tests/api/bandid.rs b/tests/api/bandid.rs new file mode 100644 index 00000000..be911f0b --- /dev/null +++ b/tests/api/bandid.rs @@ -0,0 +1,95 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +use std::str::FromStr; + +use assert_matches::assert_matches; + +use conserve::{BandId, Error}; + +#[test] +#[should_panic] +fn empty_id_not_allowed() { + BandId::new(&[]); +} + +#[test] +fn equality() { + assert_eq!(BandId::new(&[1]), BandId::new(&[1])) +} + +#[test] +fn zero() { + assert_eq!(BandId::zero().to_string(), "b0000"); +} + +#[test] +fn zero_has_no_previous() { + assert_eq!(BandId::zero().previous(), None); +} + +#[test] +fn previous_of_one_is_zero() { + assert_eq!( + BandId::zero().next_sibling().previous(), + Some(BandId::zero()) + ); +} + +#[test] +fn next_of_zero_is_one() { + assert_eq!(BandId::zero().next_sibling().to_string(), "b0001"); +} + +#[test] +fn next_of_two_is_three() { + assert_eq!(BandId::from(2).next_sibling().to_string(), "b0003"); +} + +#[test] +fn to_string() { + let band_id = BandId::new(&[20]); + assert_eq!(band_id.to_string(), "b0020"); +} + +#[test] +fn large_value_to_string() { + assert_eq!(BandId::new(&[2_000_000]).to_string(), "b2000000") +} + +#[test] +fn from_string_detects_invalid() { + assert!(BandId::from_str("").is_err()); + assert!(BandId::from_str("hello").is_err()); + assert!(BandId::from_str("b").is_err()); + assert!(BandId::from_str("b-").is_err()); + assert!(BandId::from_str("b2-").is_err()); + assert!(BandId::from_str("b-2").is_err()); + assert!(BandId::from_str("b2-1-").is_err()); + assert!(BandId::from_str("b2--1").is_err()); + assert!(BandId::from_str("beta").is_err()); + assert!(BandId::from_str("b-eta").is_err()); + assert!(BandId::from_str("b-1eta").is_err()); + assert!(BandId::from_str("b-1-eta").is_err()); +} + +#[test] +fn from_string_valid() { + assert_eq!(BandId::from_str("b0001").unwrap().to_string(), "b0001"); + assert_eq!(BandId::from_str("b123456").unwrap().to_string(), "b123456"); +} + +#[test] +fn dashes_are_no_longer_valid() { + // Versions prior to 23.2 accepted bandids with dashes, but never + // used them. + let err = BandId::from_str("b0001-0100-0234").unwrap_err(); + assert_matches!(err, Error::InvalidVersion { .. }); +} + +#[test] +fn to_string_respects_padding() { + let s = format!("{:<10}", BandId::from(42)); + assert_eq!(s.len(), 10); + assert_eq!(s, "b0042 "); +} diff --git a/tests/api/main.rs b/tests/api/main.rs index cb6032fc..1148bf2f 100644 --- a/tests/api/main.rs +++ b/tests/api/main.rs @@ -14,6 +14,7 @@ mod apath; mod backup; +mod bandid; mod blockhash; mod damaged; mod delete; From a53e8880841f5c11e03e32fbaf119ad334f37593 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 08:28:04 -0800 Subject: [PATCH 07/86] Revert "Try to avoid redundant CI on PRs (#204)" This reverts commit 33995b9298819a696de54c196c812d7485a38a0e. --- .github/workflows/rust.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 252e93f8..42ce2481 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,11 +1,6 @@ name: Rust -on: - push: - branches: - - "main" - - "releases/*" - pull_request: +on: [push, pull_request] # see https://matklad.github.io/2021/09/04/fast-rust-builds.html env: From 74d033a000f587b78f5b01274dcb495bd9ca7353 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 08:41:29 -0800 Subject: [PATCH 08/86] Metrics for local transport --- src/transport/local.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/transport/local.rs b/src/transport/local.rs index 2548e916..3df800f5 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -1,4 +1,4 @@ -// Copyright 2020 Martin Pool. +// Copyright 2020-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -19,6 +19,7 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use bytes::Bytes; +use metrics::{counter, increment_counter}; use crate::transport::{DirEntry, Metadata, Transport}; @@ -50,6 +51,7 @@ impl Transport for LocalTransport { // let's pass them back as lossy UTF-8 so they can be reported at a higher level, for // example during validation. let full_path = self.full_path(relpath); + increment_counter!("conserve.local_transport.read_dirs"); Ok(Box::new(full_path.read_dir()?.map(move |de_result| { let de = de_result?; Ok(DirEntry { @@ -60,19 +62,26 @@ impl Transport for LocalTransport { } fn read_file(&self, relpath: &str) -> io::Result { + increment_counter!("conserve.local_transport.read_files"); let mut file = File::open(self.full_path(relpath))?; let estimated_len: usize = file.metadata()?.len().try_into().unwrap(); let mut out_buf = Vec::with_capacity(estimated_len); let actual_len = file.read_to_end(&mut out_buf)?; + counter!( + "conserve.local_transport.read_file_bytes", + actual_len as u64 + ); out_buf.truncate(actual_len); Ok(out_buf.into()) } fn is_file(&self, relpath: &str) -> io::Result { + increment_counter!("conserve.local_transport.metadata_reads"); Ok(self.full_path(relpath).is_file()) } fn is_dir(&self, relpath: &str) -> io::Result { + increment_counter!("conserve.local_transport.metadata_reads"); Ok(self.full_path(relpath).is_dir()) } @@ -87,6 +96,11 @@ impl Transport for LocalTransport { } fn write_file(&self, relpath: &str, content: &[u8]) -> io::Result<()> { + increment_counter!("conserve.local_transport.write_files"); + counter!( + "conserve.local_transport.write_file_bytes", + content.len() as u64 + ); let full_path = self.full_path(relpath); let dir = full_path.parent().unwrap(); let mut temp = tempfile::Builder::new() @@ -123,6 +137,7 @@ impl Transport for LocalTransport { } fn metadata(&self, relpath: &str) -> io::Result { + increment_counter!("conserve.local_transport.metadata_reads"); let fsmeta = self.root.join(relpath).metadata()?; Ok(Metadata { len: fsmeta.len(), From 6b26b7c82bfbc42806852527e8c70a721b7b9994 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 12 Feb 2023 15:21:37 -0800 Subject: [PATCH 09/86] News about new options --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 542aec66..601df464 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,8 @@ - Don't complain if unable to chown during restore; this is normal when not run as root. +- New `--log-json` global option to capture all logs, and `--metrics-json` to write out counters. + ## 23.1.1 - Fixed: User and group mappings are now cached in memory. This fixes a performance regression in From ab6910bf71d63ae70006eee7ccb9b7db43642d9a Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 15 Feb 2023 21:13:32 -0800 Subject: [PATCH 10/86] Make Entry object safe Remove Eq, PartialEq, and a method that is incompatible. They're barely used. --- src/backup.rs | 11 ++++++++++- src/entry.rs | 14 ++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 60f67fb5..53787771 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -241,7 +241,7 @@ impl BackupWriter { let apath = source_entry.apath(); let result; if let Some(basis_entry) = self.basis_index.advance_to(apath) { - if source_entry.is_unchanged_from(&basis_entry) { + if entry_metadata_unchanged(source_entry, &basis_entry) { self.stats.unmodified_files += 1; self.index_builder.push_entry(basis_entry); return Ok(Some(DiffKind::Unchanged)); @@ -446,3 +446,12 @@ impl FileCombiner { } } } +/// True if the metadata supports an assumption the file contents have +/// not changed. +fn entry_metadata_unchanged(new_entry: &E, basis_entry: &O) -> bool { + basis_entry.kind() == new_entry.kind() + && basis_entry.mtime() == new_entry.mtime() + && basis_entry.size() == new_entry.size() + && basis_entry.unix_mode() == new_entry.unix_mode() + && basis_entry.owner() == new_entry.owner() +} diff --git a/src/entry.rs b/src/entry.rs index 9a20ef6f..ecd2ca59 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -22,7 +22,7 @@ use crate::unix_mode::UnixMode; use crate::unix_time::UnixTime; use crate::*; -pub trait Entry: Debug + Eq + PartialEq { +pub trait Entry: Debug { fn apath(&self) -> &Apath; fn kind(&self) -> Kind; fn mtime(&self) -> UnixTime; @@ -30,14 +30,4 @@ pub trait Entry: Debug + Eq + PartialEq { fn symlink_target(&self) -> &Option; fn unix_mode(&self) -> UnixMode; fn owner(&self) -> Owner; - - /// True if the metadata supports an assumption the file contents have - /// not changed. - fn is_unchanged_from(&self, basis_entry: &O) -> bool { - basis_entry.kind() == self.kind() - && basis_entry.mtime() == self.mtime() - && basis_entry.size() == self.size() - && basis_entry.unix_mode() == self.unix_mode() - && basis_entry.owner() == self.owner() - } } From d1921ba4c64decbf7875642193db8053f1387fb3 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 17 Feb 2023 08:58:28 -0800 Subject: [PATCH 11/86] News of progress bars --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 601df464..8c95952d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ ## Unreleased +- Better progress bars for various operations including `validate`. + - Don't complain if unable to chown during restore; this is normal when not run as root. - New `--log-json` global option to capture all logs, and `--metrics-json` to write out counters. From 72cd975255e15f487f6f4c67d74ecc93b8a4cdce Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 17 Feb 2023 08:56:30 -0800 Subject: [PATCH 12/86] Write 0.6.3 format unless there are flags There currently are never any format flags set, but there might be soon. Add some more tests for this. --- Cargo.lock | 2 +- Cargo.toml | 2 +- NEWS.md | 4 ++++ doc/format.md | 3 ++- src/band.rs | 25 +++++++++++++++++-------- tests/api/format_flags.rs | 39 ++++++++++++++++++++++++++++++++++++--- 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6694ae5f..108e5357 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,7 +216,7 @@ dependencies = [ [[package]] name = "conserve" -version = "23.2.0-pre" +version = "23.2.0" dependencies = [ "assert_cmd", "assert_fs", diff --git a/Cargo.toml b/Cargo.toml index a680ccc6..7578eb57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-2.0" name = "conserve" readme = "README.md" repository = "https://github.com/sourcefrog/conserve/" -version = "23.2.0-pre" +version = "23.2.0" rust-version = "1.63" [[bin]] diff --git a/NEWS.md b/NEWS.md index 8c95952d..abcc5ce5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,10 @@ - New `--log-json` global option to capture all logs, and `--metrics-json` to write out counters. +- New internal non-breaking format change: backups (in the band header) can now declare some + format flags needed to read the backup correctly. If any format flags are set then at least + Conserve 23.2.0 is needed to read the backup. + ## 23.1.1 - Fixed: User and group mappings are now cached in memory. This fixes a performance regression in diff --git a/doc/format.md b/doc/format.md index 969bcaba..3a0f3de5 100644 --- a/doc/format.md +++ b/doc/format.md @@ -154,7 +154,8 @@ The head file contains: - `band_format_version`: The minimum program version to correctly read this band. - `format_flags`: A list of strings indicating capabilities required to read - this band correctly. Only present from 23.2 onwards. + this band correctly. If this is set and non-empty, then the `band_format_version` + must be at least 23.2.0. ### Band tail file diff --git a/src/band.rs b/src/band.rs index 4ae190d4..5b5566ad 100644 --- a/src/band.rs +++ b/src/band.rs @@ -35,10 +35,6 @@ use crate::*; static INDEX_DIR: &str = "i"; -/// Band format-compatibility. Bands written out by this program, can only be -/// read correctly by versions equal or later than the stated version. -pub const BAND_FORMAT_VERSION: &str = "23.2.0"; - /// Per-band format flags. pub mod flags { use std::borrow::Cow; @@ -47,7 +43,7 @@ pub mod flags { pub static DEFAULT: &[Cow<'static, str>] = &[]; /// All the flags understood by this version of Conserve. - pub static SUPPORTED: &[Cow<'static, str>] = &[]; + pub static SUPPORTED: &[&'static str] = &[]; } /// Describes how to select a band from an archive. @@ -62,7 +58,7 @@ pub enum BandSelectionPolicy { } fn band_version_requirement() -> semver::VersionReq { - semver::VersionReq::parse(&format!("<={BAND_FORMAT_VERSION}")).unwrap() + semver::VersionReq::parse(&format!("<={}", crate::VERSION)).unwrap() } fn band_version_supported(version: &str) -> bool { @@ -139,6 +135,9 @@ impl Band { archive: &Archive, format_flags: &[Cow<'static, str>], ) -> Result { + format_flags + .iter() + .for_each(|f| assert!(flags::SUPPORTED.contains(&f.as_ref()), "unknown flag {f:?}")); let band_id = archive .last_band_id()? .map_or_else(BandId::zero, |b| b.next_sibling()); @@ -147,9 +146,14 @@ impl Band { .create_dir("") .and_then(|()| transport.create_dir(INDEX_DIR)) .map_err(|source| Error::CreateBand { source })?; + let band_format_version = if format_flags.is_empty() { + Some("0.6.3".to_owned()) + } else { + Some("23.2.0".to_owned()) + }; let head = Head { start_time: OffsetDateTime::now_utc().unix_timestamp(), - band_format_version: Some(BAND_FORMAT_VERSION.to_owned()), + band_format_version, format_flags: format_flags.into(), }; write_json(&transport, BAND_HEAD_FILENAME, &head)?; @@ -192,7 +196,7 @@ impl Band { let unsupported_flags = head .format_flags .iter() - .filter(|f| !flags::SUPPORTED.contains(f)) + .filter(|f| !flags::SUPPORTED.contains(&f.as_ref())) .cloned() .collect_vec(); if !unsupported_flags.is_empty() { @@ -231,6 +235,11 @@ impl Band { &self.band_id } + /// Get the minimum supported version for this band. + pub fn band_format_version(&self) -> Option<&str> { + self.head.band_format_version.as_deref() + } + /// Get the format flags in this band, from [flags]. pub fn format_flags(&self) -> &[Cow<'static, str>] { &self.head.format_flags diff --git a/tests/api/format_flags.rs b/tests/api/format_flags.rs index 4fd296db..9a4b1160 100644 --- a/tests/api/format_flags.rs +++ b/tests/api/format_flags.rs @@ -8,14 +8,47 @@ use assert_matches::assert_matches; use conserve::test_fixtures::ScratchArchive; use conserve::*; +#[test] +// This can be updated if/when Conserve does start writing some flags by default. +fn default_format_flags_are_empty() { + let af = ScratchArchive::new(); + + let orig_band = Band::create(&af).unwrap(); + let flags = orig_band.format_flags(); + assert!(flags.is_empty(), "{flags:?}"); + + let band = Band::open(&af, orig_band.id()).unwrap(); + println!("{band:?}"); + assert!(band.format_flags().is_empty()); + + assert_eq!(band.band_format_version(), Some("0.6.3")); + // TODO: When we do support some flags, check that the minimum version is 23.2. +} + +#[test] +#[should_panic(expected = "unknown flag \"wibble\"")] +fn unknown_format_flag_panics_in_create() { + let af = ScratchArchive::new(); + let _ = Band::create_with_flags(&af, &["wibble".into()]); +} + #[test] fn unknown_format_flag_fails_to_open() { let af = ScratchArchive::new(); - let orig_band = Band::create_with_flags(&af, &["wibble".into()]).unwrap(); - assert_eq!(orig_band.format_flags(), ["wibble"]); + // Make the bandhead by hand because the library prevents writing invalid flags. + af.transport().create_dir("b0000").unwrap(); + let head = serde_json::json! ({ + "start_time": 1676651990, + "band_format_version": "23.2.0", + "format_flags": ["wibble"] + }); + af.transport() + .sub_transport("b0000") + .write_file("BANDHEAD", &serde_json::to_vec(&head).unwrap()) + .unwrap(); - let err = Band::open(&af, orig_band.id()).unwrap_err(); + let err = Band::open(&af, &BandId::zero()).unwrap_err(); println!("{err}"); assert_matches!(err, Error::UnsupportedBandFormatFlags { .. }); assert!(err From 744c0788c6acef9f7a9a9b301107329a345fe43f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 19 Feb 2023 18:29:49 -0800 Subject: [PATCH 13/86] Replace our UnixTime with time::OffsetDateTime --- src/entry.rs | 5 ++-- src/index.rs | 14 +++++----- src/live_tree.rs | 8 +++--- src/restore.rs | 11 ++++---- src/unix_time.rs | 56 +++++++++++++++++++-------------------- tests/api/live_tree.rs | 15 +++-------- tests/api/old_archives.rs | 15 +++++++---- tests/api/restore.rs | 13 +++------ 8 files changed, 64 insertions(+), 73 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index ecd2ca59..04499621 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -16,16 +16,17 @@ use std::fmt::Debug; +use time::OffsetDateTime; + use crate::kind::Kind; use crate::owner::Owner; use crate::unix_mode::UnixMode; -use crate::unix_time::UnixTime; use crate::*; pub trait Entry: Debug { fn apath(&self) -> &Apath; fn kind(&self) -> Kind; - fn mtime(&self) -> UnixTime; + fn mtime(&self) -> OffsetDateTime; fn size(&self) -> Option; fn symlink_target(&self) -> &Option; fn unix_mode(&self) -> UnixMode; diff --git a/src/index.rs b/src/index.rs index a81a8f53..e8430daa 100644 --- a/src/index.rs +++ b/src/index.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use std::vec; use metrics::{counter, increment_counter}; +use time::OffsetDateTime; use tracing::error; use crate::compress::snappy::{Compressor, Decompressor}; @@ -30,7 +31,7 @@ use crate::stats::{IndexReadStats, IndexWriterStats}; use crate::transport::local::LocalTransport; use crate::transport::Transport; use crate::unix_mode::UnixMode; -use crate::unix_time::UnixTime; +use crate::unix_time::FromUnixAndNanos; use crate::*; pub const MAX_ENTRIES_PER_HUNK: usize = 1000; @@ -99,11 +100,8 @@ impl Entry for IndexEntry { } #[inline] - fn mtime(&self) -> UnixTime { - UnixTime { - secs: self.mtime, - nanosecs: self.mtime_nanos, - } + fn mtime(&self) -> OffsetDateTime { + OffsetDateTime::from_unix_seconds_and_nanos(self.mtime, self.mtime_nanos) } /// Size of the file, if it is a file. None for directories and symlinks. @@ -139,8 +137,8 @@ impl IndexEntry { kind: source.kind(), addrs: Vec::new(), target: source.symlink_target().clone(), - mtime: mtime.secs, - mtime_nanos: mtime.nanosecs, + mtime: mtime.unix_timestamp(), + mtime_nanos: mtime.nanosecond(), unix_mode: source.unix_mode(), owner: source.owner(), } diff --git a/src/live_tree.rs b/src/live_tree.rs index 015b213d..cfce2bcf 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -18,12 +18,12 @@ use std::fs; use std::io::ErrorKind; use std::path::{Path, PathBuf}; +use time::OffsetDateTime; use tracing::{error, warn}; use crate::owner::Owner; use crate::stats::LiveTreeIterStats; use crate::unix_mode::UnixMode; -use crate::unix_time::UnixTime; use crate::*; /// A real tree on the filesystem, for use as a backup source or restore destination. @@ -56,7 +56,7 @@ impl LiveTree { pub struct LiveEntry { apath: Apath, kind: Kind, - mtime: UnixTime, + mtime: OffsetDateTime, size: Option, symlink_target: Option, unix_mode: UnixMode, @@ -97,7 +97,7 @@ impl Entry for LiveEntry { self.kind } - fn mtime(&self) -> UnixTime { + fn mtime(&self) -> OffsetDateTime { self.mtime } diff --git a/src/restore.rs b/src/restore.rs index 0ef78e0b..3b3a4ed3 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -21,6 +21,7 @@ use std::{fs, time::Instant}; use filetime::set_file_handle_times; #[cfg(unix)] use filetime::set_symlink_file_times; +use time::OffsetDateTime; #[allow(unused_imports)] use tracing::{error, warn}; @@ -30,7 +31,7 @@ use crate::io::{directory_is_empty, ensure_dir_exists}; use crate::progress::{Bar, Progress}; use crate::stats::RestoreStats; use crate::unix_mode::UnixMode; -use crate::unix_time::UnixTime; +use crate::unix_time::ToFileTime; use crate::*; /// Description of how to restore a tree. @@ -150,7 +151,7 @@ pub struct RestoreTree { path: PathBuf, dir_unix_modes: Vec<(PathBuf, UnixMode)>, - dir_mtimes: Vec<(PathBuf, UnixTime)>, + dir_mtimes: Vec<(PathBuf, OffsetDateTime)>, } impl RestoreTree { @@ -192,7 +193,7 @@ impl RestoreTree { } } for (path, time) in self.dir_mtimes { - if let Err(err) = filetime::set_file_mtime(&path, time.into()) { + if let Err(err) = filetime::set_file_mtime(&path, time.to_file_time()) { error!("Failed to set directory mtime on {path:?}: {err}"); } } @@ -230,7 +231,7 @@ impl RestoreTree { std::io::copy(content, &mut restore_file).map_err(restore_err)?; restore_file.flush().map_err(restore_err)?; - let mtime = Some(source_entry.mtime().into()); + let mtime = Some(source_entry.mtime().to_file_time()); set_file_handle_times(&restore_file, mtime, mtime).map_err(|source| { Error::RestoreModificationTime { path: path.clone(), @@ -263,7 +264,7 @@ impl RestoreTree { if let Err(source) = unix_fs::symlink(target, &path) { return Err(Error::Restore { path, source }); } - let mtime = entry.mtime().into(); + let mtime = entry.mtime().to_file_time(); if let Err(source) = set_symlink_file_times(&path, mtime, mtime) { return Err(Error::RestoreModificationTime { path, source }); } diff --git a/src/unix_time.rs b/src/unix_time.rs index e9793813..bad52bd8 100644 --- a/src/unix_time.rs +++ b/src/unix_time.rs @@ -12,43 +12,41 @@ // GNU General Public License for more details. //! Times relative to the Unix epoch. +//! +//! In particular, glue between [filetime] and [time]. use filetime::FileTime; +use time::OffsetDateTime; -use std::convert::From; -use std::time::{SystemTime, UNIX_EPOCH}; +pub(crate) trait FromUnixAndNanos { + fn from_unix_seconds_and_nanos(unix_seconds: i64, nanoseconds: u32) -> Self; +} + +impl FromUnixAndNanos for OffsetDateTime { + fn from_unix_seconds_and_nanos(unix_seconds: i64, nanoseconds: u32) -> Self { + OffsetDateTime::from_unix_timestamp(unix_seconds) + .unwrap() + .replace_nanosecond(nanoseconds) + .unwrap() + } +} -/// A Unix time, as seconds since 1970 UTC, plus fractional nanoseconds. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct UnixTime { - /// Whole seconds after (or if negative, before) 1 Jan 1970 UTC. - pub secs: i64, - /// Fractional nanoseconds. - pub nanosecs: u32, +pub(crate) trait ToOffsetDateTime { + fn to_offset_date_time(&self) -> OffsetDateTime; } -impl From for UnixTime { - fn from(t: SystemTime) -> UnixTime { - if let Ok(after) = t.duration_since(UNIX_EPOCH) { - UnixTime { - secs: after.as_secs() as i64, - nanosecs: after.subsec_nanos(), - } - } else { - let before = UNIX_EPOCH.duration_since(t).unwrap(); - let mut secs = -(before.as_secs() as i64); - let mut nanosecs = before.subsec_nanos(); - if nanosecs > 0 { - secs -= 1; - nanosecs = 1_000_000_000 - nanosecs; - } - UnixTime { secs, nanosecs } - } +impl ToOffsetDateTime for FileTime { + fn to_offset_date_time(&self) -> OffsetDateTime { + OffsetDateTime::from_unix_seconds_and_nanos(self.unix_seconds(), self.nanoseconds()) } } -impl From for FileTime { - fn from(t: UnixTime) -> FileTime { - FileTime::from_unix_time(t.secs, t.nanosecs) +pub(crate) trait ToFileTime { + fn to_file_time(&self) -> FileTime; +} + +impl ToFileTime for OffsetDateTime { + fn to_file_time(&self) -> FileTime { + FileTime::from_unix_time(self.unix_timestamp(), self.nanosecond()) } } diff --git a/tests/api/live_tree.rs b/tests/api/live_tree.rs index b2a599c2..645144be 100644 --- a/tests/api/live_tree.rs +++ b/tests/api/live_tree.rs @@ -1,4 +1,4 @@ -// Copyright 2021, 2022 Martin Pool. +// Copyright 2021-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -11,7 +11,6 @@ // GNU General Public License for more details. use pretty_assertions::assert_eq; -use regex::Regex; use conserve::test_fixtures::TreeFixture; use conserve::*; @@ -53,15 +52,9 @@ fn list_simple_directory() { ); let repr = format!("{:?}", &result[6]); - - let re_str = r#"LiveEntry \{ apath: Apath\("/jam/apricot"\), kind: "#.to_owned() - + r#"File, mtime: UnixTime \{ [^)]* \}, size: Some\(8\), symlink_target: None, "# - + r#"unix_mode: UnixMode\((Some\([0-9]+\)\)|None), "# - + r#"owner: Owner \{ user: (Some\("[a-z_][a-z0-9_-]*[$]?"\)|None), "# - + r#"group: (Some\("[a-z_][a-z0-9_-]*[$]?"\)|None) \} \}"#; - - let re = Regex::new(&re_str).unwrap(); - assert!(re.is_match(&repr)); + println!("{repr}"); + assert!(repr.starts_with("LiveEntry {")); + assert!(repr.contains("Apath(\"/jam/apricot\")")); // TODO: Somehow get the stats out of the iterator. // assert_eq!(source_iter.stats.directories_visited, 4); diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index e1c4030d..058ca93a 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -22,8 +22,8 @@ use assert_fs::TempDir; use predicates::prelude::*; use pretty_assertions::assert_eq; -use conserve::unix_time::UnixTime; use conserve::*; +use time::OffsetDateTime; use tracing_test::traced_test; const MINIMAL_ARCHIVE_VERSIONS: &[&str] = &["0.6.0", "0.6.10", "0.6.2", "0.6.3", "0.6.9", "0.6.17"]; @@ -154,20 +154,25 @@ fn restore_old_archive() { // Check that mtimes are restored. The sub-second times are not tested // because their behavior might vary depending on the local filesystem. - let file_mtime = UnixTime::from( + let file_mtime = OffsetDateTime::from( metadata(dest.child("hello").path()) .unwrap() .modified() .unwrap(), ); - assert_eq!(file_mtime.secs, 1592266523, "mtime not restored correctly"); - let dir_mtime = UnixTime::from( + assert_eq!( + file_mtime.unix_timestamp(), + 1592266523, + "mtime not restored correctly" + ); + + let dir_mtime = OffsetDateTime::from( metadata(dest.child("subdir").path()) .unwrap() .modified() .unwrap(), ); - assert_eq!(dir_mtime.secs, 1592266523,); + assert_eq!(dir_mtime.unix_timestamp(), 1592266523); } } diff --git a/tests/api/restore.rs b/tests/api/restore.rs index c6703de4..d1aabd6e 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -1,4 +1,4 @@ -// Copyright 2015, 2016, 2017, 2019, 2020 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -21,7 +21,6 @@ use tempfile::TempDir; use conserve::test_fixtures::ScratchArchive; use conserve::test_fixtures::TreeFixture; -use conserve::unix_time::UnixTime; use conserve::*; #[test] @@ -123,12 +122,8 @@ fn restore_symlink() { let srcdir = TreeFixture::new(); srcdir.create_symlink("symlink", "target"); - let years_ago = UnixTime { - secs: 189216000, - nanosecs: 0, - }; - let mtime: FileTime = years_ago.into(); - set_symlink_file_times(srcdir.path().join("symlink"), mtime, mtime).unwrap(); + let years_ago = FileTime::from_unix_time(189216000, 0); + set_symlink_file_times(srcdir.path().join("symlink"), years_ago, years_ago).unwrap(); backup(&af, &srcdir.live_tree(), &Default::default()).unwrap(); @@ -138,7 +133,7 @@ fn restore_symlink() { let restored_symlink_path = restore_dir.path().join("symlink"); let sym_meta = symlink_metadata(&restored_symlink_path).unwrap(); assert!(sym_meta.file_type().is_symlink()); - assert_eq!(UnixTime::from(sym_meta.modified().unwrap()), years_ago); + assert_eq!(FileTime::from(sym_meta.modified().unwrap()), years_ago); assert_eq!( read_link(&restored_symlink_path).unwrap(), PathBuf::from("target") From 242ee56c1eb69ebac9ec3316432f3cc4405ec14a Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 12 Feb 2023 16:29:44 -0800 Subject: [PATCH 14/86] Refactor Restore RestoreTree seems like an unnecessary object in Rust: let's just use functions --- src/lib.rs | 2 +- src/restore.rs | 274 ++++++++++++++++++++----------------------- tests/api/restore.rs | 8 +- 3 files changed, 133 insertions(+), 151 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 982a494b..0fc18716 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,7 +69,7 @@ pub use crate::kind::Kind; pub use crate::live_tree::{LiveEntry, LiveTree}; pub use crate::merge::{MergeTrees, MergedEntryKind}; pub use crate::misc::bytes_to_human_mb; -pub use crate::restore::{restore, RestoreOptions, RestoreTree}; +pub use crate::restore::{restore, RestoreOptions}; pub use crate::show::{show_diff, show_versions, ShowVersionsOptions}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats}; pub use crate::stored_tree::StoredTree; diff --git a/src/restore.rs b/src/restore.rs index 3b3a4ed3..c99cda69 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -64,15 +64,21 @@ impl Default for RestoreOptions { /// Restore a selected version, or by default the latest, to a destination directory. pub fn restore( archive: &Archive, - destination_path: &Path, + destination: &Path, options: &RestoreOptions, ) -> Result { let st = archive.open_stored_tree(options.band_selection.clone())?; - let mut rt = if options.overwrite { - RestoreTree::create_overwrite(destination_path) - } else { - RestoreTree::create(destination_path) - }?; + if let Err(source) = ensure_dir_exists(destination) { + return Err(Error::Restore { + path: destination.to_owned(), + source, + }); + } + if !options.overwrite && !directory_is_empty(destination)? { + return Err(Error::DestinationNotEmpty { + path: destination.to_owned(), + }); + } let mut stats = RestoreStats::default(); let mut bytes_done = 0; let bar = Bar::new(); @@ -92,6 +98,7 @@ pub fn restore( options.only_subtree.clone().unwrap_or_else(Apath::root), options.exclude.clone(), )?; + let mut deferrals = Vec::new(); for entry in entry_iter { if options.print_filenames { if options.long_listing { @@ -99,187 +106,158 @@ pub fn restore( } else { println!("{}", entry.apath()); } - } - if !options.print_filenames { + } else { bar.post(Progress::Restore { filename: entry.apath().to_string(), bytes_done, }); } - if let Err(err) = match entry.kind() { + let path = destination.join(&entry.apath[1..]); + match entry.kind() { Kind::Dir => { stats.directories += 1; - rt.copy_dir(&entry) + if let Err(err) = fs::create_dir_all(&path) { + if err.kind() != io::ErrorKind::AlreadyExists { + error!(?path, ?err, "Failed to create directory"); + stats.errors += 1; + continue; + } + } + deferrals.push(DirDeferral { + path, + unix_mode: entry.unix_mode(), + mtime: entry.mtime(), + }) } Kind::File => { stats.files += 1; - let result = rt.copy_file(&entry, &st).map(|s| stats += s); - if let Some(bytes) = entry.size() { - bytes_done += bytes; + match copy_file(path.clone(), &entry, &st) { + Err(err) => { + error!(?err, ?path, "Failed to restore file"); + stats.errors += 1; + } + Ok(s) => { + if let Some(bytes) = entry.size() { + bytes_done += bytes; + } + stats += s; + } } - result } Kind::Symlink => { stats.symlinks += 1; - rt.copy_symlink(&entry) + if let Err(err) = restore_symlink(path.clone(), &entry) { + error!(?path, ?err, "Failed to restore symlink"); + stats.errors += 1; + } } Kind::Unknown => { stats.unknown_kind += 1; - // TODO: Perhaps eventually we could backup and restore pipes, - // sockets, etc. Or at least count them. For now, silently skip. - // https://github.com/sourcefrog/conserve/issues/82 - continue; + warn!(apath = ?entry.apath(), "Unknown file kind"); } - } { - error!( - "error restoring {apath}: {err}", - apath = entry.apath().to_string() - ); - stats.errors += 1; - continue; - } + }; } - stats += rt.finish()?; + stats += apply_deferrals(&deferrals)?; stats.elapsed = start.elapsed(); // TODO: Merge in stats from the tree iter and maybe the source tree? Ok(stats) } -/// A write-only tree on the filesystem, as a restore destination. -#[derive(Debug)] -pub struct RestoreTree { +/// Recorded changes to apply to directories after all their contents +/// have been applied. +/// +/// For example we might want to make the directory read-only, but we +/// shouldn't do that until we added all the children. +struct DirDeferral { path: PathBuf, - - dir_unix_modes: Vec<(PathBuf, UnixMode)>, - dir_mtimes: Vec<(PathBuf, OffsetDateTime)>, + unix_mode: UnixMode, + mtime: OffsetDateTime, } -impl RestoreTree { - fn new(path: PathBuf) -> RestoreTree { - RestoreTree { - path, - dir_mtimes: Vec::new(), - dir_unix_modes: Vec::new(), - } - } - - /// Create a RestoreTree. - /// - /// The destination must either not yet exist, or be an empty directory. - pub fn create>(path: P) -> Result { - let path = path.into(); - match ensure_dir_exists(&path).and_then(|()| directory_is_empty(&path)) { - Err(source) => Err(Error::Restore { path, source }), - Ok(true) => Ok(RestoreTree::new(path)), - Ok(false) => Err(Error::DestinationNotEmpty { path }), - } - } - - /// Create a RestoreTree, even if the destination directory is not empty. - pub fn create_overwrite(path: &Path) -> Result { - Ok(RestoreTree::new(path.to_path_buf())) - } - - fn rooted_path(&self, apath: &Apath) -> PathBuf { - // Remove initial slash so that the apath is relative to the destination. - self.path.join(&apath[1..]) - } - - fn finish(self) -> Result { - #[cfg(unix)] - for (path, unix_mode) in self.dir_unix_modes { - if let Err(err) = unix_mode.set_permissions(&path) { - error!("Failed to set directory permissions on {path:?}: {err}"); - } +fn apply_deferrals(deferrals: &[DirDeferral]) -> Result { + let mut stats = RestoreStats::default(); + for DirDeferral { + path, + unix_mode, + mtime, + } in deferrals + { + if let Err(err) = unix_mode.set_permissions(path) { + error!(?path, ?err, "Failed to set directory permissions"); + stats.errors += 1; } - for (path, time) in self.dir_mtimes { - if let Err(err) = filetime::set_file_mtime(&path, time.to_file_time()) { - error!("Failed to set directory mtime on {path:?}: {err}"); - } + if let Err(err) = filetime::set_file_mtime(path, (*mtime).to_file_time()) { + error!(?path, ?err, "Failed to set directory mtime"); + stats.errors += 1; } - Ok(RestoreStats::default()) } + Ok(stats) +} - fn copy_dir(&mut self, entry: &E) -> Result<()> { - let path = self.rooted_path(entry.apath()); - if let Err(source) = fs::create_dir_all(&path) { - if source.kind() != io::ErrorKind::AlreadyExists { - return Err(Error::Restore { path, source }); - } - } - self.dir_mtimes.push((path.clone(), entry.mtime())); - self.dir_unix_modes.push((path, entry.unix_mode())); - Ok(()) - } +/// Copy in the contents of a file from another tree. +fn copy_file( + path: PathBuf, + source_entry: &R::Entry, + from_tree: &R, +) -> Result { + let restore_err = |source| Error::Restore { + path: path.clone(), + source, + }; + let mut stats = RestoreStats::default(); + let mut restore_file = File::create(&path).map_err(restore_err)?; + // TODO: Read one block at a time: don't pull all the contents into memory. + let content = &mut from_tree.file_contents(source_entry)?; + stats.uncompressed_file_bytes = + std::io::copy(content, &mut restore_file).map_err(restore_err)?; + restore_file.flush().map_err(restore_err)?; - /// Copy in the contents of a file from another tree. - fn copy_file( - &mut self, - source_entry: &R::Entry, - from_tree: &R, - ) -> Result { - let path = self.rooted_path(source_entry.apath()); - let restore_err = |source| Error::Restore { + let mtime = Some(source_entry.mtime().to_file_time()); + set_file_handle_times(&restore_file, mtime, mtime).map_err(|source| { + Error::RestoreModificationTime { path: path.clone(), source, - }; - let mut stats = RestoreStats::default(); - let mut restore_file = File::create(&path).map_err(restore_err)?; - // TODO: Read one block at a time: don't pull all the contents into memory. - let content = &mut from_tree.file_contents(source_entry)?; - stats.uncompressed_file_bytes = - std::io::copy(content, &mut restore_file).map_err(restore_err)?; - restore_file.flush().map_err(restore_err)?; - - let mtime = Some(source_entry.mtime().to_file_time()); - set_file_handle_times(&restore_file, mtime, mtime).map_err(|source| { - Error::RestoreModificationTime { - path: path.clone(), - source, - } - })?; - - // Restore permissions only if there are mode bits stored in the archive - if let Err(err) = source_entry.unix_mode().set_permissions(&path) { - error!(?path, ?err, "Error restoring unix permissions"); - stats.errors += 1; } + })?; - // Restore ownership if possible. - // TODO: Stats and warnings if a user or group is specified in the index but - // does not exist on the local system. - if let Err(err) = &source_entry.owner().set_owner(&path) { - error!(?path, ?err, "Error restoring ownership"); - stats.errors += 1; - } - // TODO: Accumulate more stats. - Ok(stats) + // Restore permissions only if there are mode bits stored in the archive + if let Err(err) = source_entry.unix_mode().set_permissions(&path) { + error!(?path, ?err, "Error restoring unix permissions"); + stats.errors += 1; } - #[cfg(unix)] - fn copy_symlink(&mut self, entry: &E) -> Result<()> { - use std::os::unix::fs as unix_fs; - if let Some(ref target) = entry.symlink_target() { - let path = self.rooted_path(entry.apath()); - if let Err(source) = unix_fs::symlink(target, &path) { - return Err(Error::Restore { path, source }); - } - let mtime = entry.mtime().to_file_time(); - if let Err(source) = set_symlink_file_times(&path, mtime, mtime) { - return Err(Error::RestoreModificationTime { path, source }); - } - } else { - // TODO: Treat as an error. - error!("No target in symlink entry {:?}", entry.apath()); - } - Ok(()) + // Restore ownership if possible. + // TODO: Stats and warnings if a user or group is specified in the index but + // does not exist on the local system. + if let Err(err) = &source_entry.owner().set_owner(&path) { + error!(?path, ?err, "Error restoring ownership"); + stats.errors += 1; } + // TODO: Accumulate more stats. + Ok(stats) +} - #[cfg(not(unix))] - fn copy_symlink(&mut self, entry: &E) -> Result<()> { - // TODO: Add a test with a canned index containing a symlink, and expect - // it cannot be restored on Windows and can be on Unix. - warn!("Can't restore symlinks on non-Unix: {}", entry.apath()); - Ok(()) +#[cfg(unix)] +fn restore_symlink(path: PathBuf, entry: &E) -> Result<()> { + use std::os::unix::fs as unix_fs; + if let Some(ref target) = entry.symlink_target() { + if let Err(source) = unix_fs::symlink(target, &path) { + return Err(Error::Restore { path, source }); + } + let mtime = entry.mtime().to_file_time(); + if let Err(source) = set_symlink_file_times(&path, mtime, mtime) { + return Err(Error::RestoreModificationTime { path, source }); + } + } else { + error!(apath = ?entry.apath(), "No target in symlink entry"); } + Ok(()) +} + +#[cfg(not(unix))] +fn restore_symlink(_restore_path: &Path, entry: &E) -> Result<()> { + // TODO: Add a test with a canned index containing a symlink, and expect + // it cannot be restored on Windows and can be on Unix. + warn!("Can't restore symlinks on non-Unix: {}", entry.apath()); + Ok(()) } diff --git a/tests/api/restore.rs b/tests/api/restore.rs index d1aabd6e..8ea7dc63 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -70,8 +70,12 @@ pub fn decline_to_overwrite() { af.store_two_versions(); let destdir = TreeFixture::new(); destdir.create_file("existing"); - let restore_err_str = RestoreTree::create(destdir.path().to_owned()) - .unwrap_err() + let options = RestoreOptions { + ..RestoreOptions::default() + }; + assert!(!options.overwrite, "overwrite is false by default"); + let restore_err_str = restore(&af, destdir.path(), &options) + .expect_err("restore should fail if the destination exists") .to_string(); assert!(restore_err_str.contains("Destination directory not empty")); } From 00cc50c244bde1a5fc9e502fa13586cb2817d902 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 14 Feb 2023 20:56:20 -0800 Subject: [PATCH 15/86] Fix Windows build --- src/restore.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/restore.rs b/src/restore.rs index c99cda69..b4f5e2c8 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -146,7 +146,7 @@ pub fn restore( } Kind::Symlink => { stats.symlinks += 1; - if let Err(err) = restore_symlink(path.clone(), &entry) { + if let Err(err) = restore_symlink(&path, &entry) { error!(?path, ?err, "Failed to restore symlink"); stats.errors += 1; } @@ -238,15 +238,21 @@ fn copy_file( } #[cfg(unix)] -fn restore_symlink(path: PathBuf, entry: &E) -> Result<()> { +fn restore_symlink(path: &Path, entry: &E) -> Result<()> { use std::os::unix::fs as unix_fs; if let Some(ref target) = entry.symlink_target() { - if let Err(source) = unix_fs::symlink(target, &path) { - return Err(Error::Restore { path, source }); + if let Err(source) = unix_fs::symlink(target, path) { + return Err(Error::Restore { + path: path.to_owned(), + source, + }); } let mtime = entry.mtime().to_file_time(); - if let Err(source) = set_symlink_file_times(&path, mtime, mtime) { - return Err(Error::RestoreModificationTime { path, source }); + if let Err(source) = set_symlink_file_times(path, mtime, mtime) { + return Err(Error::RestoreModificationTime { + path: path.to_owned(), + source, + }); } } else { error!(apath = ?entry.apath(), "No target in symlink entry"); From fc75b5e137b1fab30555a1ff50e73151e29fff25 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 12 Feb 2023 16:39:54 -0800 Subject: [PATCH 16/86] Metrics during restore --- src/restore.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/restore.rs b/src/restore.rs index b4f5e2c8..46cb5b1c 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -21,6 +21,7 @@ use std::{fs, time::Instant}; use filetime::set_file_handle_times; #[cfg(unix)] use filetime::set_symlink_file_times; +use metrics::{counter, increment_counter}; use time::OffsetDateTime; #[allow(unused_imports)] use tracing::{error, warn}; @@ -116,6 +117,7 @@ pub fn restore( match entry.kind() { Kind::Dir => { stats.directories += 1; + increment_counter!("conserve.restore.dirs"); if let Err(err) = fs::create_dir_all(&path) { if err.kind() != io::ErrorKind::AlreadyExists { error!(?path, ?err, "Failed to create directory"); @@ -131,6 +133,7 @@ pub fn restore( } Kind::File => { stats.files += 1; + increment_counter!("conserve.restore.files"); match copy_file(path.clone(), &entry, &st) { Err(err) => { error!(?err, ?path, "Failed to restore file"); @@ -146,6 +149,7 @@ pub fn restore( } Kind::Symlink => { stats.symlinks += 1; + increment_counter!("conserve.restore.symlinks"); if let Err(err) = restore_symlink(&path, &entry) { error!(?path, ?err, "Failed to restore symlink"); stats.errors += 1; @@ -208,8 +212,9 @@ fn copy_file( let mut restore_file = File::create(&path).map_err(restore_err)?; // TODO: Read one block at a time: don't pull all the contents into memory. let content = &mut from_tree.file_contents(source_entry)?; - stats.uncompressed_file_bytes = - std::io::copy(content, &mut restore_file).map_err(restore_err)?; + let len = std::io::copy(content, &mut restore_file).map_err(restore_err)?; + stats.uncompressed_file_bytes = len; + counter!("conserve.restore.file_bytes", len); restore_file.flush().map_err(restore_err)?; let mtime = Some(source_entry.mtime().to_file_time()); From 544ee6ce2c6aa14408f22ad8caff548d034127d8 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Mon, 20 Feb 2023 07:32:47 -0800 Subject: [PATCH 17/86] Clippy --- src/band.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/band.rs b/src/band.rs index 5b5566ad..90525575 100644 --- a/src/band.rs +++ b/src/band.rs @@ -43,7 +43,7 @@ pub mod flags { pub static DEFAULT: &[Cow<'static, str>] = &[]; /// All the flags understood by this version of Conserve. - pub static SUPPORTED: &[&'static str] = &[]; + pub static SUPPORTED: &[&str] = &[]; } /// Describes how to select a band from an archive. From 00c8af0b5067dc86d2860fba59d09c7346537cba Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 23 Feb 2023 21:03:59 -0800 Subject: [PATCH 18/86] Fix progress bar for block deletion They were using an enumerate visited by a par_iter, which produces a chaotic ordering. --- src/archive.rs | 12 +++++++----- src/progress.rs | 1 + src/progress/term.rs | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 7d895798..ee64631d 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -299,16 +299,18 @@ impl Archive { }); } + let blocks_done: AtomicUsize = AtomicUsize::new(0); + let start = Instant::now(); let error_count = unref .par_iter() - .enumerate() - .inspect(|(blocks_done, _hash)| { + .filter(|block_hash| { bar.post(Progress::DeleteBlocks { - blocks_done: *blocks_done, + blocks_done: blocks_done.fetch_add(1, Ordering::Relaxed), + start, total_blocks: unref_count, - }) + }); + block_dir.delete_block(block_hash).is_err() }) - .filter(|(_, block_hash)| block_dir.delete_block(block_hash).is_err()) .count(); stats.deletion_errors += error_count; stats.deleted_block_count += unref_count - error_count; diff --git a/src/progress.rs b/src/progress.rs index 92b299b3..41175174 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -86,6 +86,7 @@ pub enum Progress { DeleteBlocks { blocks_done: usize, total_blocks: usize, + start: Instant, }, ListBlocks { count: usize, diff --git a/src/progress/term.rs b/src/progress/term.rs index f9d4017a..52253b1f 100644 --- a/src/progress/term.rs +++ b/src/progress/term.rs @@ -124,10 +124,12 @@ impl nutmeg::Model for Progress { Progress::DeleteBlocks { blocks_done, total_blocks, + start, } => format!( - "Delete blocks: {}/{}...", + "Delete blocks: {}/{}, {eta} remaining...", blocks_done.separate_with_commas(), total_blocks.separate_with_commas(), + eta = estimate_remaining(start, *blocks_done, *total_blocks), ), Progress::ListBlocks { count } => format!( "List blocks: {count}...", From 488109077ba66b312995461f0ea43f76e477168c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 9 Mar 2023 20:17:06 -0800 Subject: [PATCH 19/86] Add --json-changes and EntryChange concept (#208) * Print restored files from a callback Add an EntryChange object These are returned from callbacks as files are handled. Can be serialized to write out json changes * Changes contain both old and new metadata * Also use ChangeCallback from restore * Rename backup option to change_callback * impl Display for EntryChange * Add restore --changes-json * Refactor diff to use EntryChange * Move diff printing into bin * Add diff --json * Move BackupStats to backup.rs * Tidy up * Better json repr from changes * Fix tests: user/group currently only on Unix * Clean up merge code * Further simplify merge * Further simplify merge MatchedEntries is now directly an enum of Left, Right, or Both, and knows how to convert itself to an EntryChange. * Parameterize Change type * comment * Test --changes-json from backup --- Cargo.toml | 2 +- NEWS.md | 6 ++ src/backup.rs | 188 +++++++++++++++++++++++----------- src/bin/conserve.rs | 105 +++++++++++++++++-- src/blockdir.rs | 3 +- src/change.rs | 182 ++++++++++++++++++++++++++++++++ src/diff.rs | 93 ++--------------- src/errors.rs | 3 +- src/lib.rs | 20 ++-- src/merge.rs | 100 ++++++++---------- src/restore.rs | 37 +++---- src/show.rs | 11 -- src/stats.rs | 91 +++------------- src/unix_mode.rs | 2 + tests/api/backup.rs | 1 - tests/api/diff.rs | 31 +++--- tests/api/old_archives.rs | 15 ++- tests/api/restore.rs | 24 ++++- tests/cli/backup.rs | 60 ++++++++++- tests/cli/diff.rs | 135 ++++++++++-------------- tests/cli/exclude.rs | 4 +- tests/cli/main.rs | 35 +++++-- tests/cli/unix/diff.rs | 87 ++++++++++++++++ tests/cli/unix/permissions.rs | 40 ++++---- tests/expensive/changes.rs | 2 +- 25 files changed, 808 insertions(+), 469 deletions(-) create mode 100644 src/change.rs create mode 100644 tests/cli/unix/diff.rs diff --git a/Cargo.toml b/Cargo.toml index 7578eb57..886c7ae2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ snap = "1.0.0" tempfile = "3" thiserror = "1.0.19" thousands = "0.2.0" -time = { version = "0.3", features = ["local-offset"] } +time = { version = "0.3", features = ["local-offset", "serde", "serde-human-readable"] } tracing = "0.1" tracing-appender = "0.2" unix_mode = "0.1" diff --git a/NEWS.md b/NEWS.md index abcc5ce5..62d5b502 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,12 @@ format flags needed to read the backup correctly. If any format flags are set then at least Conserve 23.2.0 is needed to read the backup. +- New `--changes-json` option to `restore` and `backup`. + +- `diff` output format has changed slightly to be the same as `backup`. + +- New `diff --json`. + ## 23.1.1 - Fixed: User and group mappings are now cached in memory. This fixes a performance regression in diff --git a/src/backup.rs b/src/backup.rs index 53787771..c51208eb 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -14,42 +14,43 @@ //! Make a backup by walking a source directory and copying the contents //! into an archive. +use std::convert::TryInto; +use std::fmt; use std::io::prelude::*; -use std::{convert::TryInto, time::Instant}; +use std::time::{Duration, Instant}; +use derive_more::{Add, AddAssign}; use itertools::Itertools; use tracing::error; use crate::blockdir::Address; +use crate::change::Change; use crate::io::read_with_retries; use crate::progress::{Bar, Progress}; -use crate::stats::BackupStats; +use crate::stats::{ + write_compressed_size, write_count, write_duration, write_size, IndexWriterStats, +}; use crate::stitch::IterStitchedIndexHunks; use crate::tree::ReadTree; use crate::*; /// Configuration of how to make a backup. -#[derive(Debug, Clone)] -pub struct BackupOptions { - /// Print filenames to the UI as they're copied. - pub print_filenames: bool, - +pub struct BackupOptions<'cb> { /// Exclude these globs from the backup. pub exclude: Exclude, - /// If printing filenames, include metadata such as file permissions - pub long_listing: bool, - pub max_entries_per_hunk: usize, + + // Call this callback as each entry is successfully stored. + pub change_callback: Option>, } -impl Default for BackupOptions { - fn default() -> BackupOptions { +impl Default for BackupOptions<'_> { + fn default() -> BackupOptions<'static> { BackupOptions { - print_filenames: false, exclude: Exclude::nothing(), max_entries_per_hunk: crate::index::MAX_ENTRIES_PER_HUNK, - long_listing: false, + change_callback: None, } } } @@ -79,8 +80,6 @@ pub fn backup( let mut stats = BackupStats::default(); let bar = Bar::new(); - let mut scanned_dirs = 0; - let mut scanned_files = 0; let mut scanned_file_bytes = 0; let mut entries_new = 0; let mut entries_changed = 0; @@ -89,56 +88,40 @@ pub fn backup( let entry_iter = source.iter_entries(Apath::root(), options.exclude.clone())?; for entry_group in entry_iter.chunks(options.max_entries_per_hunk).into_iter() { for entry in entry_group { - match entry.kind() { - Kind::Dir => scanned_dirs += 1, - Kind::File => scanned_files += 1, - _ => (), - } match writer.copy_entry(&entry, source) { Err(err) => { error!(?entry, ?err, "Error copying entry to backup"); stats.errors += 1; continue; } - Ok(Some(diff_kind)) => { - if options.print_filenames && diff_kind != DiffKind::Unchanged { - if options.long_listing { - println!( - "{} {} {} {}", - diff_kind.as_sigil(), - entry.unix_mode(), - entry.owner(), - entry.apath() - ); - } else { - println!("{} {}", diff_kind.as_sigil(), entry.apath()); - } - } - match diff_kind { - DiffKind::Changed => entries_changed += 1, - DiffKind::New => entries_new += 1, - DiffKind::Unchanged => entries_unchanged += 1, + Ok(Some(entry_change)) => { + match entry_change.change { + Change::Changed { .. } => entries_changed += 1, + Change::Added { .. } => entries_new += 1, + Change::Unchanged { .. } => entries_unchanged += 1, // Deletions are not produced at the moment. - DiffKind::Deleted => (), // model.entries_deleted += 1, + Change::Deleted { .. } => (), // model.entries_deleted += 1, + } + if let Some(cb) = &options.change_callback { + cb(&entry_change)?; } } Ok(_) => {} } - if let Some(bytes) = entry.size() { - if bytes > 0 { + match entry.size() { + Some(bytes) if bytes > 0 => { scanned_file_bytes += bytes; - if !options.print_filenames { - bar.post(Progress::Backup { - filename: entry.apath().to_string(), - scanned_file_bytes, - scanned_dirs, - scanned_files, - entries_new, - entries_changed, - entries_unchanged, - }); - } + bar.post(Progress::Backup { + filename: entry.apath().to_string(), + scanned_file_bytes, + scanned_dirs: stats.directories, + scanned_files: stats.files, + entries_new, + entries_changed, + entries_unchanged, + }); } + _ => (), } } writer.flush_group()?; @@ -209,7 +192,8 @@ impl BackupWriter { /// Return an indication of whether it changed (if it's a file), or /// None for non-plain-file types where that information is not currently /// calculated. - fn copy_entry(&mut self, entry: &LiveEntry, source: &LiveTree) -> Result> { + fn copy_entry(&mut self, entry: &LiveEntry, source: &LiveTree) -> Result> { + // TODO: Emit deletions for entries in the basis not present in the source. match entry.kind() { Kind::Dir => self.copy_dir(entry), Kind::File => self.copy_file(entry, source), @@ -224,11 +208,11 @@ impl BackupWriter { } } - fn copy_dir(&mut self, source_entry: &E) -> Result> { + fn copy_dir(&mut self, source_entry: &E) -> Result> { self.stats.directories += 1; self.index_builder .push_entry(IndexEntry::metadata_from(source_entry)); - Ok(None) // TODO: See if it changed from the basis? + Ok(None) // TODO: Emit the actual change. } /// Copy in the contents of a file from another tree. @@ -236,22 +220,23 @@ impl BackupWriter { &mut self, source_entry: &LiveEntry, from_tree: &LiveTree, - ) -> Result> { + ) -> Result> { self.stats.files += 1; let apath = source_entry.apath(); let result; if let Some(basis_entry) = self.basis_index.advance_to(apath) { if entry_metadata_unchanged(source_entry, &basis_entry) { self.stats.unmodified_files += 1; + let change = Some(EntryChange::unchanged(&basis_entry)); self.index_builder.push_entry(basis_entry); - return Ok(Some(DiffKind::Unchanged)); + return Ok(change); } else { self.stats.modified_files += 1; - result = Some(DiffKind::Changed); + result = Some(EntryChange::changed(&basis_entry, source_entry)); } } else { self.stats.new_files += 1; - result = Some(DiffKind::New); + result = Some(EntryChange::added(source_entry)); } let mut read_source = from_tree.file_contents(source_entry)?; let size = source_entry.size().expect("LiveEntry has a size"); @@ -279,12 +264,13 @@ impl BackupWriter { Ok(result) } - fn copy_symlink(&mut self, source_entry: &E) -> Result> { + fn copy_symlink(&mut self, source_entry: &E) -> Result> { let target = source_entry.symlink_target().clone(); self.stats.symlinks += 1; assert!(target.is_some()); self.index_builder .push_entry(IndexEntry::metadata_from(source_entry)); + // TODO: Emit the actual change. Ok(None) } } @@ -446,8 +432,11 @@ impl FileCombiner { } } } + /// True if the metadata supports an assumption the file contents have -/// not changed. +/// not changed, without reading the file content. +/// +/// Caution: this does not check the symlink target. fn entry_metadata_unchanged(new_entry: &E, basis_entry: &O) -> bool { basis_entry.kind() == new_entry.kind() && basis_entry.mtime() == new_entry.mtime() @@ -455,3 +444,78 @@ fn entry_metadata_unchanged(new_entry: &E, basis_entry: &O) && basis_entry.unix_mode() == new_entry.unix_mode() && basis_entry.owner() == new_entry.owner() } + +#[derive(Add, AddAssign, Debug, Default, Eq, PartialEq, Clone)] +pub struct BackupStats { + // TODO: Have separate more-specific stats for backup and restore, and then + // each can have a single Display method. + // TODO: Include source file bytes, including unmodified files. + pub files: usize, + pub symlinks: usize, + pub directories: usize, + pub unknown_kind: usize, + + pub unmodified_files: usize, + pub modified_files: usize, + pub new_files: usize, + + /// Bytes that matched an existing block. + pub deduplicated_bytes: u64, + /// Bytes that were stored as new blocks, before compression. + pub uncompressed_bytes: u64, + pub compressed_bytes: u64, + + pub deduplicated_blocks: usize, + pub written_blocks: usize, + /// Blocks containing combined small files. + pub combined_blocks: usize, + + pub empty_files: usize, + pub small_combined_files: usize, + pub single_block_files: usize, + pub multi_block_files: usize, + + pub errors: usize, + + pub index_builder_stats: IndexWriterStats, + pub elapsed: Duration, +} + +impl fmt::Display for BackupStats { + fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { + write_count(w, "files:", self.files); + write_count(w, " unmodified files", self.unmodified_files); + write_count(w, " modified files", self.modified_files); + write_count(w, " new files", self.new_files); + write_count(w, "symlinks", self.symlinks); + write_count(w, "directories", self.directories); + write_count(w, "unsupported file kind", self.unknown_kind); + writeln!(w).unwrap(); + + write_count(w, "files stored:", self.new_files + self.modified_files); + write_count(w, " empty files", self.empty_files); + write_count(w, " small combined files", self.small_combined_files); + write_count(w, " single block files", self.single_block_files); + write_count(w, " multi-block files", self.multi_block_files); + writeln!(w).unwrap(); + + write_count(w, "data blocks deduplicated:", self.deduplicated_blocks); + write_size(w, " saved", self.deduplicated_bytes); + writeln!(w).unwrap(); + + write_count(w, "new data blocks written:", self.written_blocks); + write_count(w, " blocks of combined files", self.combined_blocks); + write_compressed_size(w, self.compressed_bytes, self.uncompressed_bytes); + writeln!(w).unwrap(); + + let idx = &self.index_builder_stats; + write_count(w, "new index hunks", idx.index_hunks); + write_compressed_size(w, idx.compressed_index_bytes, idx.uncompressed_index_bytes); + writeln!(w).unwrap(); + + write_count(w, "errors", self.errors); + write_duration(w, "elapsed", self.elapsed)?; + + Ok(()) + } +} diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index fc1295cd..ffa37cae 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -13,12 +13,16 @@ //! Command-line entry point for Conserve backups. +use std::cell::RefCell; use std::error::Error; +use std::fs::OpenOptions; use std::io::{BufWriter, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Instant; use clap::{Parser, Subcommand}; +use conserve::change::Change; +use conserve::progress::ProgressImpl; use conserve::trace_counter::{global_error_count, global_warn_count}; use metrics::increment_counter; #[allow(unused_imports)] @@ -65,13 +69,18 @@ enum Command { archive: String, /// Source directory to copy from. source: PathBuf, + /// Write a list of changes to this file. + #[arg(long)] + changes_json: Option, /// Print copied file names. #[arg(long, short)] verbose: bool, #[arg(long, short)] exclude: Vec, + /// Read a list of globs to exclude from this file. #[arg(long, short = 'E')] exclude_from: Vec, + /// Don't print statistics after the backup completes. #[arg(long)] no_stats: bool, /// Show permissions, owner, and group in verbose output. @@ -111,6 +120,10 @@ enum Command { exclude_from: Vec, #[arg(long)] include_unchanged: bool, + + /// Print the diff as json. + #[arg(long, short)] + json: bool, }, /// Create a new archive. @@ -154,6 +167,9 @@ enum Command { destination: PathBuf, #[arg(long, short)] backup: Option, + /// Write a list of restored files to this json file. + #[arg(long)] + changes_json: Option, #[arg(long, short)] force_overwrite: bool, #[arg(long, short)] @@ -275,20 +291,28 @@ impl Command { match self { Command::Backup { archive, - source, - verbose, + changes_json, exclude, exclude_from, - no_stats, long_listing, + no_stats, + source, + verbose, } => { let source = &LiveTree::open(source)?; let options = BackupOptions { - print_filenames: *verbose, exclude: Exclude::from_patterns_and_files(exclude, exclude_from)?, - long_listing: *long_listing, + change_callback: make_change_callback( + *verbose, + *long_listing, + &changes_json.as_deref(), + )?, ..Default::default() }; + if *long_listing || *verbose { + // TODO(CON-23): Really Nutmeg should coordinate stdout and stderr... + ProgressImpl::Null.activate() + } let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options)?; if !no_stats { info!("Backup complete.\n{stats}"); @@ -345,6 +369,7 @@ impl Command { exclude, exclude_from, include_unchanged, + json, } => { let st = stored_tree_from_opt(archive, backup)?; let lt = LiveTree::open(source)?; @@ -352,7 +377,14 @@ impl Command { exclude: Exclude::from_patterns_and_files(exclude, exclude_from)?, include_unchanged: *include_unchanged, }; - show_diff(diff(&st, <, &options)?, &mut stdout)?; + let mut bw = BufWriter::new(stdout); + for change in diff(&st, <, &options)? { + if *json { + serde_json::to_writer(&mut bw, &change)?; + } else { + writeln!(bw, "{change}")?; + } + } } Command::Gc { archive, @@ -404,6 +436,7 @@ impl Command { archive, destination, backup, + changes_json, verbose, force_overwrite, exclude, @@ -415,13 +448,19 @@ impl Command { let band_selection = band_selection_policy_from_opt(backup); let archive = Archive::open(open_transport(archive)?)?; let options = RestoreOptions { - print_filenames: *verbose, exclude: Exclude::from_patterns_and_files(exclude, exclude_from)?, only_subtree: only_subtree.clone(), band_selection, overwrite: *force_overwrite, - long_listing: *long_listing, + change_callback: make_change_callback( + *verbose, + *long_listing, + &changes_json.as_deref(), + )?, }; + if *verbose || *long_listing { + ProgressImpl::Null.activate(); + } let stats = restore(&archive, destination, &options)?; debug!("Restore complete"); if !no_stats { @@ -497,6 +536,54 @@ fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolic } } +fn make_change_callback<'a>( + print_changes: bool, + ls_long: bool, + changes_json: &Option<&Path>, +) -> Result>> { + if !print_changes && !ls_long && changes_json.is_none() { + return Ok(None); + }; + + let changes_json_writer = if let Some(path) = changes_json { + Some(RefCell::new(BufWriter::new( + OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?, + ))) + } else { + None + }; + Ok(Some(Box::new(move |entry_change| { + if matches!(entry_change.change, Change::Unchanged { .. }) { + return Ok(()); + } + if ls_long { + let change_meta = entry_change.change.primary_metadata(); + println!( + "{} {} {} {}", + entry_change.change.sigil(), + change_meta.unix_mode, + change_meta.owner, + entry_change.apath + ); + } else if print_changes { + println!("{} {}", entry_change.change.sigil(), entry_change.apath); + } + if let Some(w) = &changes_json_writer { + let mut w = w.borrow_mut(); + writeln!( + w, + "{}", + serde_json::to_string(entry_change).expect("Failed to serialize change") + )?; + } + Ok(()) + }))) +} + fn main() -> Result { let args = Args::parse(); let start_time = Instant::now(); diff --git a/src/blockdir.rs b/src/blockdir.rs index 931477af..e7806f91 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -37,11 +37,12 @@ use serde::{Deserialize, Serialize}; #[allow(unused_imports)] use tracing::{debug, error, info, warn}; +use crate::backup::BackupStats; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; use crate::kind::Kind; use crate::progress::{Bar, Progress}; -use crate::stats::{BackupStats, Sizes}; +use crate::stats::Sizes; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, ListDirNames, Transport}; use crate::*; diff --git a/src/change.rs b/src/change.rs new file mode 100644 index 00000000..9bc88b80 --- /dev/null +++ b/src/change.rs @@ -0,0 +1,182 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! A change to an entry during backup, diff, restore, etc. + +use std::fmt; + +use serde::Serialize; +use time::OffsetDateTime; + +use crate::{Apath, Entry, Kind, Owner, Result, UnixMode}; + +/// Summary of some kind of change to an entry from backup, diff, restore, etc. +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +pub struct EntryChange { + pub apath: Apath, + #[serde(flatten)] + pub change: Change, +} + +impl EntryChange { + pub fn is_unchanged(&self) -> bool { + self.change.is_unchanged() + } + + pub(crate) fn diff_metadata(a: &dyn Entry, b: &dyn Entry) -> Self { + debug_assert_eq!(a.apath(), b.apath()); + let ak = a.kind(); + // mtime is only treated as a significant change for files, because + // the behavior on directories is not consistent between Unix and + // Windows (and maybe not across filesystems even on Unix.) + if ak != b.kind() + || a.owner() != b.owner() + || a.unix_mode() != b.unix_mode() + || (ak == Kind::File && (a.size() != b.size() || a.mtime() != b.mtime())) + || (ak == Kind::Symlink && (a.symlink_target() != b.symlink_target())) + { + EntryChange::changed(a, b) + } else { + EntryChange::unchanged(a) + } + } + + pub(crate) fn added(entry: &dyn Entry) -> Self { + EntryChange { + apath: entry.apath().clone(), + change: Change::Added { + added: EntryMetadata::from(entry), + }, + } + } + + #[allow(unused)] // Never generated in backups at the moment + pub(crate) fn deleted(entry: &dyn Entry) -> Self { + EntryChange { + apath: entry.apath().clone(), + change: Change::Deleted { + deleted: EntryMetadata::from(entry), + }, + } + } + + pub(crate) fn unchanged(entry: &dyn Entry) -> Self { + EntryChange { + apath: entry.apath().clone(), + change: Change::Unchanged { + unchanged: EntryMetadata::from(entry), + }, + } + } + + pub(crate) fn changed(old: &dyn Entry, new: &dyn Entry) -> Self { + debug_assert_eq!(old.apath(), new.apath()); + EntryChange { + apath: old.apath().clone(), + change: Change::Changed { + old: EntryMetadata::from(old), + new: EntryMetadata::from(new), + }, + } + } +} + +impl fmt::Display for EntryChange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} {}", self.change.sigil(), self.apath) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +#[serde(tag = "change")] +pub enum Change { + Unchanged { unchanged: E }, + Added { added: E }, + Deleted { deleted: E }, + Changed { old: E, new: E }, +} + +impl Change { + pub fn is_unchanged(&self) -> bool { + matches!(self, Change::Unchanged { .. }) + } + + /// Return the primary metadata: the new version, unless this entry was + /// deleted in which case the old version. + pub fn primary_metadata(&self) -> &E { + match self { + Change::Unchanged { unchanged } => unchanged, + Change::Added { added } => added, + Change::Deleted { deleted } => deleted, + Change::Changed { new, .. } => new, + } + } + + pub fn sigil(&self) -> char { + match self { + Change::Unchanged { .. } => '.', + Change::Added { .. } => '+', + Change::Deleted { .. } => '-', + Change::Changed { .. } => '*', + } + } +} + +/// Metadata about a changed entry other than its apath. +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +pub struct EntryMetadata { + // TODO: Eventually unify with LiveEntry or Entry? + #[serde(flatten)] + pub kind: KindMetadata, + pub mtime: OffsetDateTime, + #[serde(flatten)] + pub owner: Owner, + pub unix_mode: UnixMode, +} + +impl From<&dyn Entry> for EntryMetadata { + fn from(entry: &dyn Entry) -> Self { + EntryMetadata { + kind: KindMetadata::from(entry), + mtime: entry.mtime(), + owner: entry.owner(), + unix_mode: entry.unix_mode(), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +#[serde(tag = "kind")] +pub enum KindMetadata { + File { size: u64 }, + Dir, + Symlink { target: String }, +} + +impl From<&dyn Entry> for KindMetadata { + fn from(entry: &dyn Entry) -> Self { + match entry.kind() { + Kind::File => KindMetadata::File { + size: entry.size().unwrap(), + }, + Kind::Dir => KindMetadata::Dir, + Kind::Symlink => KindMetadata::Symlink { + target: entry.symlink_target().clone().unwrap(), + }, + Kind::Unknown => panic!("unexpected Kind::Unknown on {:?}", entry.apath()), + } + } +} + +/// A callback when a changed entry is visited, e.g. during a backup. +pub type ChangeCallback<'cb> = Box Result<()> + 'cb>; diff --git a/src/diff.rs b/src/diff.rs index aa561502..ec5c08a8 100644 --- a/src/diff.rs +++ b/src/diff.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -15,20 +15,16 @@ //! //! See also [conserve::show_diff] to format the diff as text. -use std::fmt; - use readahead_iterator::IntoReadahead; use crate::*; -use DiffKind::*; -use Kind::*; -use MergedEntryKind::*; - #[derive(Debug)] pub struct DiffOptions { pub exclude: Exclude, pub include_unchanged: bool, + // TODO: An option to filter to a subtree? + // TODO: Optionally compare all the content? } impl Default for DiffOptions { @@ -40,95 +36,22 @@ impl Default for DiffOptions { } } -/// The overall state of change of an entry. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum DiffKind { - Unchanged, - New, - Deleted, - Changed, -} - -impl DiffKind { - pub fn as_sigil(self) -> char { - match self { - Unchanged => '.', - New => '+', - Deleted => '-', - Changed => '*', - } - } -} - -#[derive(Debug, Eq, PartialEq)] -pub struct DiffEntry { - pub apath: Apath, - pub kind: DiffKind, -} - -impl fmt::Display for DiffEntry { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}\t{}", self.kind.as_sigil(), self.apath) - } -} - /// Generate an iter of per-entry diffs between two trees. pub fn diff( st: &StoredTree, lt: &LiveTree, options: &DiffOptions, -) -> Result> { +) -> Result> { let readahead = 1000; - let include_unchanged: bool = options.include_unchanged; - // TODO: Take an option for the subtree? + let include_unchanged: bool = options.include_unchanged; // Copy out to avoid lifetime problems in the callback let ait = st .iter_entries(Apath::root(), options.exclude.clone())? .readahead(readahead); let bit = lt .iter_entries(Apath::root(), options.exclude.clone())? - .filter(|le| le.kind() != Unknown) + .filter(|le| le.kind() != Kind::Unknown) .readahead(readahead); Ok(MergeTrees::new(ait, bit) - .map(diff_merged_entry) - .filter(move |de: &DiffEntry| include_unchanged || de.kind != DiffKind::Unchanged)) -} - -fn diff_merged_entry(me: merge::MergedEntry) -> DiffEntry -where - AE: Entry, - BE: Entry, -{ - let apath = me.apath; - match me.kind { - Both(ae, be) => diff_common_entry(ae, be, apath), - LeftOnly(_) => DiffEntry { - kind: Deleted, - apath, - }, - RightOnly(_) => DiffEntry { kind: New, apath }, - } -} - -fn diff_common_entry(ae: AE, be: BE, apath: Apath) -> DiffEntry -where - AE: Entry, - BE: Entry, -{ - // TODO: Actually compare content, if requested. - // TODO: Skip Kind::Unknown. - let ak = ae.kind(); - if ak != be.kind() - || (ak == File && (ae.mtime() != be.mtime() || ae.size() != be.size())) - || (ak == Symlink && (ae.symlink_target() != be.symlink_target())) - { - DiffEntry { - kind: Changed, - apath, - } - } else { - DiffEntry { - kind: Unchanged, - apath, - } - } + .map(|me| me.to_entry_change()) + .filter(move |c: &EntryChange| include_unchanged || !c.is_unchanged())) } diff --git a/src/errors.rs b/src/errors.rs index e34ede08..3cb12f80 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -282,8 +282,9 @@ pub enum Error { #[error("Unsupported URL scheme {:?}", scheme)] UrlScheme { scheme: String }, - #[error("Failed to serialize problem")] + #[error("Failed to serialize object")] SerializeError { + #[from] #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, diff --git a/src/lib.rs b/src/lib.rs index 0fc18716..8cf2ba7a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ mod band; pub mod bandid; mod blockdir; pub mod blockhash; +pub mod change; pub mod compress; mod diff; mod entry; @@ -53,13 +54,13 @@ pub mod validate; pub use crate::apath::Apath; pub use crate::archive::Archive; pub use crate::archive::DeleteOptions; -pub use crate::backup::{backup, BackupOptions}; -pub use crate::band::Band; -pub use crate::band::BandSelectionPolicy; +pub use crate::backup::{backup, BackupOptions, BackupStats}; +pub use crate::band::{Band, BandSelectionPolicy}; pub use crate::bandid::BandId; pub use crate::blockdir::BlockDir; pub use crate::blockhash::BlockHash; -pub use crate::diff::{diff, DiffEntry, DiffKind, DiffOptions}; +pub use crate::change::{ChangeCallback, EntryChange}; +pub use crate::diff::{diff, DiffOptions}; pub use crate::entry::Entry; pub use crate::errors::Error; pub use crate::excludes::Exclude; @@ -67,14 +68,16 @@ pub use crate::gc_lock::GarbageCollectionLock; pub use crate::index::{IndexEntry, IndexRead, IndexWriter}; pub use crate::kind::Kind; pub use crate::live_tree::{LiveEntry, LiveTree}; -pub use crate::merge::{MergeTrees, MergedEntryKind}; +pub use crate::merge::MergeTrees; pub use crate::misc::bytes_to_human_mb; +pub use crate::owner::Owner; pub use crate::restore::{restore, RestoreOptions}; -pub use crate::show::{show_diff, show_versions, ShowVersionsOptions}; -pub use crate::stats::{BackupStats, DeleteStats, RestoreStats}; +pub use crate::show::{show_versions, ShowVersionsOptions}; +pub use crate::stats::{DeleteStats, RestoreStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; +pub use crate::unix_mode::UnixMode; pub use crate::validate::ValidateOptions; pub type Result = std::result::Result; @@ -112,3 +115,6 @@ static BAND_TAIL_FILENAME: &str = "BANDTAIL"; /// Length of the binary content hash. pub(crate) const BLAKE_HASH_SIZE_BYTES: usize = 64; + +/// A callback when an entry is visited. +pub type EntryCallback<'cb> = Box; diff --git a/src/merge.rs b/src/merge.rs index fb641f61..10686820 100644 --- a/src/merge.rs +++ b/src/merge.rs @@ -1,4 +1,4 @@ -// Copyright 2018, 2019, 2020, 2021 Martin Pool. +// Copyright 2018-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -19,30 +19,38 @@ use std::cmp::Ordering; use crate::*; +/// When merging entries from two trees a particular apath might +/// be present in either or both trees. +/// +/// Unlike the [Change] struct, this contains the full entry rather than +/// just metadata, and in particular will contain the block addresses for +/// [IndexEntry]. #[derive(Debug, PartialEq, Eq)] -pub enum MergedEntryKind +pub enum MatchedEntries where AE: Entry, BE: Entry, { - LeftOnly(AE), - RightOnly(BE), + Left(AE), + Right(BE), Both(AE, BE), } -use self::MergedEntryKind::*; - -#[derive(Debug, PartialEq, Eq)] -pub struct MergedEntry +impl MatchedEntries where AE: Entry, BE: Entry, { - pub apath: Apath, - pub kind: MergedEntryKind, + pub(crate) fn to_entry_change(&self) -> EntryChange { + match self { + MatchedEntries::Both(ae, be) => EntryChange::diff_metadata(ae, be), + MatchedEntries::Left(ae) => EntryChange::deleted(ae), + MatchedEntries::Right(be) => EntryChange::added(be), + } + } } -/// Zip together entries from two trees, into an iterator of MergedEntryKind. +/// Zip together entries from two trees, into an iterator of [MatchedEntries]. /// /// Note that at present this only says whether files are absent from either /// side, not whether there is a content difference. @@ -55,8 +63,9 @@ where { ait: AIT, bit: BIT, - // Read in advance entries from A and B. + /// Peeked next entry from [ait]. na: Option, + /// Peeked next entry from [bit]. nb: Option, } @@ -84,65 +93,39 @@ where AIT: Iterator, BIT: Iterator, { - type Item = MergedEntry; + type Item = MatchedEntries; fn next(&mut self) -> Option { - // TODO: Stats about the merge. - let ait = &mut self.ait; - let bit = &mut self.bit; - // Preload next-A and next-B, if they're not already - // loaded. - // - // TODO: Perhaps use `Peekable` instead of keeping a readahead here? + // Preload next-A and next-B, if they're not already loaded. if self.na.is_none() { - self.na = ait.next(); + self.na = self.ait.next(); } if self.nb.is_none() { - self.nb = bit.next(); + self.nb = self.bit.next(); } - if self.na.is_none() { - if self.nb.is_none() { - None - } else { - let tb = self.nb.take().unwrap(); - Some(MergedEntry { - apath: tb.apath().clone(), - kind: RightOnly(tb), - }) - } - } else if self.nb.is_none() { - let ta = self.na.take().unwrap(); - Some(MergedEntry { - apath: ta.apath().clone(), - kind: LeftOnly(ta), - }) - } else { - let pa = self.na.as_ref().unwrap().apath().clone(); - let pb = self.nb.as_ref().unwrap().apath().clone(); - match pa.cmp(&pb) { - Ordering::Equal => Some(MergedEntry { - apath: pa, - kind: Both(self.na.take().unwrap(), self.nb.take().unwrap()), - }), - Ordering::Less => Some(MergedEntry { - apath: pa, - kind: LeftOnly(self.na.take().unwrap()), - }), - Ordering::Greater => Some(MergedEntry { - apath: pb, - kind: RightOnly(self.nb.take().unwrap()), - }), - } + match (&self.na, &self.nb) { + (None, None) => None, + (Some(_a), None) => Some(MatchedEntries::Left(self.na.take().unwrap())), + (None, Some(_b)) => Some(MatchedEntries::Right(self.nb.take().unwrap())), + (Some(a), Some(b)) => match a.apath().cmp(b.apath()) { + Ordering::Equal => Some(MatchedEntries::Both( + self.na.take().unwrap(), + self.nb.take().unwrap(), + )), + Ordering::Less => Some(MatchedEntries::Left(self.na.take().unwrap())), + Ordering::Greater => Some(MatchedEntries::Right(self.nb.take().unwrap())), + }, } } } #[cfg(test)] mod tests { - use super::MergedEntryKind::*; use crate::test_fixtures::*; use crate::*; + use super::MatchedEntries; + #[test] fn merge_entry_trees() { let ta = TreeFixture::new(); @@ -157,9 +140,8 @@ mod tests { ) .collect::>(); assert_eq!(di.len(), 1); - assert_eq!(di[0].apath, "/"); - match &di[0].kind { - Both(ae, be) => { + match &di[0] { + MatchedEntries::Both(ae, be) => { assert_eq!(ae.kind(), Kind::Dir); assert_eq!(be.kind(), Kind::Dir); assert_eq!(ae.apath(), "/"); diff --git a/src/restore.rs b/src/restore.rs index 46cb5b1c..1f4a75a1 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -36,28 +36,27 @@ use crate::unix_time::ToFileTime; use crate::*; /// Description of how to restore a tree. -#[derive(Debug)] -pub struct RestoreOptions { - pub print_filenames: bool, +// #[derive(Debug)] +pub struct RestoreOptions<'cb> { pub exclude: Exclude, /// Restore only this subdirectory. pub only_subtree: Option, pub overwrite: bool, // The band to select, or by default the last complete one. pub band_selection: BandSelectionPolicy, - /// If printing filenames, include metadata such as file permissions - pub long_listing: bool, + + // Call this callback as each entry is successfully restored. + pub change_callback: Option>, } -impl Default for RestoreOptions { +impl Default for RestoreOptions<'_> { fn default() -> Self { RestoreOptions { - print_filenames: false, overwrite: false, band_selection: BandSelectionPolicy::LatestClosed, exclude: Exclude::nothing(), only_subtree: None, - long_listing: false, + change_callback: None, } } } @@ -101,18 +100,10 @@ pub fn restore( )?; let mut deferrals = Vec::new(); for entry in entry_iter { - if options.print_filenames { - if options.long_listing { - println!("{} {} {}", entry.unix_mode(), entry.owner(), entry.apath()); - } else { - println!("{}", entry.apath()); - } - } else { - bar.post(Progress::Restore { - filename: entry.apath().to_string(), - bytes_done, - }); - } + bar.post(Progress::Restore { + filename: entry.apath().to_string(), + bytes_done, + }); let path = destination.join(&entry.apath[1..]); match entry.kind() { Kind::Dir => { @@ -138,6 +129,7 @@ pub fn restore( Err(err) => { error!(?err, ?path, "Failed to restore file"); stats.errors += 1; + continue; } Ok(s) => { if let Some(bytes) = entry.size() { @@ -153,6 +145,7 @@ pub fn restore( if let Err(err) = restore_symlink(&path, &entry) { error!(?path, ?err, "Failed to restore symlink"); stats.errors += 1; + continue; } } Kind::Unknown => { @@ -160,6 +153,10 @@ pub fn restore( warn!(apath = ?entry.apath(), "Unknown file kind"); } }; + if let Some(cb) = options.change_callback.as_ref() { + // Since we only restore to empty directories they're all added. + cb(&EntryChange::added(&entry))?; + } } stats += apply_deferrals(&deferrals)?; stats.elapsed = start.elapsed(); diff --git a/src/show.rs b/src/show.rs index 844358bc..29ccc2ca 100644 --- a/src/show.rs +++ b/src/show.rs @@ -149,14 +149,3 @@ pub fn show_entry_names>( } Ok(()) } - -pub fn show_diff>(diff: D, w: &mut dyn Write) -> Result<()> { - // TODO: Consider whether the actual files have changed. - // TODO: Summarize diff. - // TODO: Optionally include unchanged files. - let mut bw = BufWriter::new(w); - for de in diff { - writeln!(bw, "{de}")?; - } - Ok(()) -} diff --git a/src/stats.rs b/src/stats.rs index c79ce5cc..8ac5fdca 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -32,11 +32,15 @@ fn ratio(uncompressed: u64, compressed: u64) -> f64 { } } -fn write_size>(w: &mut fmt::Formatter<'_>, label: &str, value: I) { +pub(crate) fn write_size>(w: &mut fmt::Formatter<'_>, label: &str, value: I) { writeln!(w, "{:>12} MB {}", mb_string(value.into()), label).unwrap(); } -fn write_compressed_size(w: &mut fmt::Formatter<'_>, compressed: u64, uncompressed: u64) { +pub(crate) fn write_compressed_size( + w: &mut fmt::Formatter<'_>, + compressed: u64, + uncompressed: u64, +) { write_size(w, "uncompressed", uncompressed); write_size( w, @@ -45,7 +49,7 @@ fn write_compressed_size(w: &mut fmt::Formatter<'_>, compressed: u64, uncompress ); } -fn write_count>(w: &mut fmt::Formatter<'_>, label: &str, value: I) { +pub(crate) fn write_count>(w: &mut fmt::Formatter<'_>, label: &str, value: I) { writeln!( w, "{:>12} {}", @@ -55,7 +59,11 @@ fn write_count>(w: &mut fmt::Formatter<'_>, label: &str, value: I .unwrap(); } -fn write_duration(w: &mut fmt::Formatter<'_>, label: &str, duration: Duration) -> fmt::Result { +pub(crate) fn write_duration( + w: &mut fmt::Formatter<'_>, + label: &str, + duration: Duration, +) -> fmt::Result { writeln!(w, "{:>12} {}", duration_to_hms(duration), label) } @@ -121,81 +129,6 @@ impl fmt::Display for RestoreStats { } } -#[derive(Add, AddAssign, Debug, Default, Eq, PartialEq, Clone)] -pub struct BackupStats { - // TODO: Have separate more-specific stats for backup and restore, and then - // each can have a single Display method. - // TODO: Include source file bytes, including unmodified files. - pub files: usize, - pub symlinks: usize, - pub directories: usize, - pub unknown_kind: usize, - - pub unmodified_files: usize, - pub modified_files: usize, - pub new_files: usize, - - /// Bytes that matched an existing block. - pub deduplicated_bytes: u64, - /// Bytes that were stored as new blocks, before compression. - pub uncompressed_bytes: u64, - pub compressed_bytes: u64, - - pub deduplicated_blocks: usize, - pub written_blocks: usize, - /// Blocks containing combined small files. - pub combined_blocks: usize, - - pub empty_files: usize, - pub small_combined_files: usize, - pub single_block_files: usize, - pub multi_block_files: usize, - - pub errors: usize, - - pub index_builder_stats: IndexWriterStats, - pub elapsed: Duration, -} - -impl fmt::Display for BackupStats { - fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { - write_count(w, "files:", self.files); - write_count(w, " unmodified files", self.unmodified_files); - write_count(w, " modified files", self.modified_files); - write_count(w, " new files", self.new_files); - write_count(w, "symlinks", self.symlinks); - write_count(w, "directories", self.directories); - write_count(w, "unsupported file kind", self.unknown_kind); - writeln!(w).unwrap(); - - write_count(w, "files stored:", self.new_files + self.modified_files); - write_count(w, " empty files", self.empty_files); - write_count(w, " small combined files", self.small_combined_files); - write_count(w, " single block files", self.single_block_files); - write_count(w, " multi-block files", self.multi_block_files); - writeln!(w).unwrap(); - - write_count(w, "data blocks deduplicated:", self.deduplicated_blocks); - write_size(w, " saved", self.deduplicated_bytes); - writeln!(w).unwrap(); - - write_count(w, "new data blocks written:", self.written_blocks); - write_count(w, " blocks of combined files", self.combined_blocks); - write_compressed_size(w, self.compressed_bytes, self.uncompressed_bytes); - writeln!(w).unwrap(); - - let idx = &self.index_builder_stats; - write_count(w, "new index hunks", idx.index_hunks); - write_compressed_size(w, idx.compressed_index_bytes, idx.uncompressed_index_bytes); - writeln!(w).unwrap(); - - write_count(w, "errors", self.errors); - write_duration(w, "elapsed", self.elapsed)?; - - Ok(()) - } -} - #[derive(Add, AddAssign, Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct DeleteStats { pub deleted_band_count: usize, diff --git a/src/unix_mode.rs b/src/unix_mode.rs index dc6c409e..26f24cdd 100644 --- a/src/unix_mode.rs +++ b/src/unix_mode.rs @@ -89,6 +89,7 @@ impl UnixMode { } } } + impl fmt::Display for UnixMode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // Convert to string. Since the file type bits are stripped, there will @@ -103,6 +104,7 @@ impl fmt::Display for UnixMode { } } } + impl From for UnixMode { fn from(mode: u32) -> Self { Self(Some(mode & MODE_BITS)) diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 5d954c44..7471ff31 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -123,7 +123,6 @@ pub fn backup_more_excludes() { let source = srcdir.live_tree(); let options = BackupOptions { exclude, - print_filenames: false, ..Default::default() }; let stats = backup(&af, &source, &options).expect("backup"); diff --git a/tests/api/diff.rs b/tests/api/diff.rs index 5c0409d8..52865461 100644 --- a/tests/api/diff.rs +++ b/tests/api/diff.rs @@ -1,4 +1,4 @@ -// Copyright 2021 Martin Pool. +// Copyright 2021-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ use conserve::test_fixtures::{ScratchArchive, TreeFixture}; use conserve::*; +use itertools::Itertools; #[test] fn diff_unchanged() { @@ -31,28 +32,20 @@ fn diff_unchanged() { include_unchanged: true, ..DiffOptions::default() }; - let des: Vec = diff(&st, <, &options).unwrap().collect(); - assert_eq!(des.len(), 2); // Root directory and the file "/thing". - assert_eq!( - des[0], - DiffEntry { - apath: "/".into(), - kind: DiffKind::Unchanged, - } - ); - assert_eq!( - des[1], - DiffEntry { - apath: "/thing".into(), - kind: DiffKind::Unchanged, - } - ); + let changes: Vec = diff(&st, <, &options).unwrap().collect(); + dbg!(&changes); + assert_eq!(changes.len(), 2); // Root directory and the file "/thing". + assert_eq!(changes[0].apath, "/"); + assert!(changes[0].is_unchanged()); + assert_eq!(changes[1].apath, "/thing"); + assert!(changes[1].is_unchanged()); // Excluding unchanged elements let options = DiffOptions { include_unchanged: false, ..DiffOptions::default() }; - - assert_eq!(diff(&st, <, &options).unwrap().count(), 0); + let changes = diff(&st, <, &options).unwrap().collect_vec(); + println!("changes with include_unchanged=false:\n{changes:#?}"); + assert_eq!(changes.len(), 0); } diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index 058ca93a..07d961e6 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -13,6 +13,7 @@ //! Read archives written by older versions. +use std::cell::RefCell; use std::collections::HashSet; use std::fs::{self, metadata, read_dir}; use std::path::Path; @@ -208,16 +209,26 @@ fn restore_modify_backup() { .expect("overwrite file"); let new_archive = Archive::open_path(&new_archive_path).expect("Open new archive"); + let emitted = RefCell::new(Vec::new()); let backup_stats = backup( &new_archive, &LiveTree::open(working_tree.path()).unwrap(), &BackupOptions { - print_filenames: true, + change_callback: Some(Box::new(|change| { + emitted + .borrow_mut() + .push((change.change.sigil(), change.apath.to_string())); + Ok(()) + })), ..Default::default() }, ) .expect("Backup modified tree"); + // Check the visited files passed to the callbacks. + let emitted = emitted.into_inner(); + dbg!(&emitted); + // Expected results for files: // "/empty" is empty and new // "/subdir/subfile" is modified @@ -230,6 +241,8 @@ fn restore_modify_backup() { working_tree.child(path).path().metadata().unwrap() ); } + assert!(emitted.contains(&('+', "/empty".to_owned()))); + assert!(emitted.contains(&('*', "/subdir/subfile".to_owned()))); assert_eq!(backup_stats.files, 3); assert!( diff --git a/tests/api/restore.rs b/tests/api/restore.rs index 8ea7dc63..fd99f146 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -12,6 +12,7 @@ //! Tests focussed on restore. +use std::cell::RefCell; #[cfg(unix)] use std::fs::{read_link, symlink_metadata}; use std::path::PathBuf; @@ -28,12 +29,31 @@ fn simple_restore() { let af = ScratchArchive::new(); af.store_two_versions(); let destdir = TreeFixture::new(); - - let options = RestoreOptions::default(); let restore_archive = Archive::open_path(af.path()).unwrap(); + let restored_names = RefCell::new(Vec::new()); + let options = RestoreOptions { + change_callback: Some(Box::new(|entry_change| { + restored_names.borrow_mut().push(entry_change.apath.clone()); + Ok(()) + })), + ..Default::default() + }; let stats = restore(&restore_archive, destdir.path(), &options).expect("restore"); assert_eq!(stats.files, 3); + let mut expected_names = vec![ + "/", + "/hello", + "/hello2", + "/link", + "/subdir", + "/subdir/subfile", + ]; + if !SYMLINKS_SUPPORTED { + expected_names.retain(|n| *n != "/link"); + } + drop(options); + assert_eq!(restored_names.into_inner(), expected_names); let dest = &destdir.path(); assert!(dest.join("hello").is_file()); diff --git a/tests/cli/backup.rs b/tests/cli/backup.rs index 8102e21c..6afa4810 100644 --- a/tests/cli/backup.rs +++ b/tests/cli/backup.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2016, 2017, 2018, 2019, 2020, 2021, 2022 Martin Pool. +// Copyright 2016-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -11,7 +11,12 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. +use std::fs::read_to_string; + use assert_cmd::prelude::*; +use assert_fs::NamedTempFile; +use indoc::indoc; +use serde_json::Deserializer; use conserve::test_fixtures::{ScratchArchive, TreeFixture}; @@ -24,6 +29,55 @@ fn backup_verbose() { src.create_dir("subdir"); src.create_file("subdir/a"); src.create_file("subdir/b"); + let changes_json = NamedTempFile::new("changes.json").unwrap(); + + run_conserve() + .args(["backup", "--no-stats", "-v"]) + .arg(af.path()) + .arg(src.path()) + .arg("--changes-json") + .arg(changes_json.path()) + .assert() + .success() + .stdout(indoc! { " + + /subdir/a + + /subdir/b + "}); + + let changes_json = read_to_string(changes_json.path()).unwrap(); + println!("{changes_json}"); + let changes: Vec = Deserializer::from_str(&changes_json) + .into_iter::() + .map(Result::unwrap) + .collect(); + assert_eq!(changes.len(), 2); + assert_eq!(changes[0]["apath"], "/subdir/a"); + assert_eq!(changes[0]["change"], "Added"); + assert_eq!(changes[0]["added"]["kind"], "File"); + assert_eq!(changes[1]["apath"], "/subdir/b"); + assert_eq!(changes[1]["change"], "Added"); + assert_eq!(changes[1]["added"]["kind"], "File"); +} + +#[test] +fn verbose_backup_does_not_print_unchanged_files() { + let af = ScratchArchive::new(); + let src = TreeFixture::new(); + src.create_file("a"); + src.create_file("b"); + + run_conserve() + .args(["backup", "--no-stats", "-v"]) + .arg(af.path()) + .arg(src.path()) + .assert() + .success() + .stdout(indoc! { " + + /a + + /b + "}); + + src.create_file_with_contents("b", b"new b contents"); run_conserve() .args(["backup", "--no-stats", "-v"]) @@ -31,5 +85,7 @@ fn backup_verbose() { .arg(src.path()) .assert() .success() - .stdout("+ /subdir/a\n+ /subdir/b\n"); + .stdout(indoc! { " + * /b + "}); } diff --git a/tests/cli/diff.rs b/tests/cli/diff.rs index 0bcc8d7b..9d990ffb 100644 --- a/tests/cli/diff.rs +++ b/tests/cli/diff.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2021 Martin Pool. +// Copyright 2021-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -13,12 +13,12 @@ //! Test `conserve diff`. -use std::fs; - use assert_cmd::prelude::*; +use indoc::indoc; use predicates::prelude::*; use conserve::test_fixtures::{ScratchArchive, TreeFixture}; +use serde_json::Value; use crate::run_conserve; @@ -36,21 +36,6 @@ fn setup() -> (ScratchArchive, TreeFixture) { (af, tf) } -#[cfg(unix)] -fn setup_symlink() -> (ScratchArchive, TreeFixture) { - let af = ScratchArchive::new(); - let tf = TreeFixture::new(); - tf.create_dir("subdir"); - tf.create_symlink("subdir/link", "target"); - run_conserve() - .arg("backup") - .arg(af.path()) - .arg(tf.path()) - .assert() - .success(); - (af, tf) -} - #[test] fn no_changes() { let (af, tf) = setup(); @@ -71,7 +56,7 @@ fn no_changes() { .arg(tf.path()) .assert() .success() - .stdout(".\t/\n.\t/hello.c\n.\t/subdir\n") + .stdout(". /\n. /hello.c\n. /subdir\n") .stderr(predicate::str::is_empty()); } @@ -79,7 +64,8 @@ fn no_changes() { fn add_entries() { let (af, tf) = setup(); tf.create_dir("src"); - tf.create_file_with_contents("src/new.rs", b"pub fn main() {}"); + let new_rs_content = b"pub fn main() {}"; + tf.create_file_with_contents("src/new.rs", new_rs_content); run_conserve() .arg("diff") @@ -87,8 +73,39 @@ fn add_entries() { .arg(tf.path()) .assert() .success() - .stdout("+\t/src\n+\t/src/new.rs\n") + .stdout(indoc! {" + + /src + + /src/new.rs + "}) + .stderr(predicate::str::is_empty()); + + // Inspect json diff + let command = run_conserve() + .args(["diff", "-j"]) + .arg(af.path()) + .arg(tf.path()) + .assert() + .success() .stderr(predicate::str::is_empty()); + let diff_json = &command.get_output().stdout; + println!("{}", std::str::from_utf8(diff_json).unwrap()); + let diff = serde_json::Deserializer::from_slice(diff_json) + .into_iter::() + .collect::, _>>() + .unwrap(); + println!("{diff:#?}"); + assert_eq!(diff.len(), 2); + assert_eq!(diff[0]["apath"], "/src"); + assert_eq!(diff[0]["added"]["kind"], "Dir"); + assert_eq!(diff[0]["added"]["size"], Value::Null); + assert!(diff[0]["added"]["mtime"].is_string()); + // User/group currently only added on Unix. + // assert!(diff[0]["added"]["user"].is_string()); + // assert!(diff[0]["added"]["group"].is_string()); + assert_eq!(diff[1]["apath"], "/src/new.rs"); + assert_eq!(diff[1]["added"]["kind"], "File"); + assert_eq!(diff[1]["added"]["size"], new_rs_content.len()); + assert!(diff[1]["added"]["mtime"].is_string()); } #[test] @@ -102,7 +119,7 @@ fn remove_file() { .arg(tf.path()) .assert() .success() - .stdout("-\t/hello.c\n") + .stdout("- /hello.c\n") .stderr(predicate::str::is_empty()); run_conserve() @@ -112,7 +129,7 @@ fn remove_file() { .arg(tf.path()) .assert() .success() - .stdout(".\t/\n-\t/hello.c\n.\t/subdir\n") + .stdout(". /\n- /hello.c\n. /subdir\n") .stderr(predicate::str::is_empty()); } @@ -128,7 +145,9 @@ fn change_kind() { .arg(tf.path()) .assert() .success() - .stdout("*\t/subdir\n") + .stdout(indoc! {" + * /subdir + "}) .stderr(predicate::str::is_empty()); run_conserve() @@ -138,7 +157,11 @@ fn change_kind() { .arg(tf.path()) .assert() .success() - .stdout(".\t/\n.\t/hello.c\n*\t/subdir\n") + .stdout(indoc! {" + . / + . /hello.c + * /subdir + "}) .stderr(predicate::str::is_empty()); } @@ -154,59 +177,9 @@ fn change_file_content() { .arg(tf.path()) .assert() .success() - .stdout("*\t/hello.c\n") - .stderr(predicate::str::is_empty()); - - run_conserve() - .arg("diff") - .arg("--include-unchanged") - .arg(af.path()) - .arg(tf.path()) - .assert() - .success() - .stdout(".\t/\n*\t/hello.c\n.\t/subdir\n") - .stderr(predicate::str::is_empty()); -} - -#[cfg(unix)] -#[test] -pub fn symlink_unchanged() { - let (af, tf) = setup_symlink(); - - run_conserve() - .arg("diff") - .arg(af.path()) - .arg(tf.path()) - .assert() - .success() - .stdout("") - .stderr(predicate::str::is_empty()); - - run_conserve() - .arg("diff") - .arg("--include-unchanged") - .arg(af.path()) - .arg(tf.path()) - .assert() - .success() - .stdout(".\t/\n.\t/subdir\n.\t/subdir/link\n") - .stderr(predicate::str::is_empty()); -} - -#[cfg(unix)] -#[test] -pub fn symlink_changed() { - let (af, tf) = setup_symlink(); - fs::remove_file(tf.path().join("subdir/link")).unwrap(); - tf.create_symlink("subdir/link", "newtarget"); - - run_conserve() - .arg("diff") - .arg(af.path()) - .arg(tf.path()) - .assert() - .success() - .stdout("*\t/subdir/link\n") + .stdout(indoc! {" + * /hello.c + "}) .stderr(predicate::str::is_empty()); run_conserve() @@ -216,6 +189,10 @@ pub fn symlink_changed() { .arg(tf.path()) .assert() .success() - .stdout(".\t/\n.\t/subdir\n*\t/subdir/link\n") + .stdout(indoc! {" + . / + * /hello.c + . /subdir + "}) .stderr(predicate::str::is_empty()); } diff --git a/tests/cli/exclude.rs b/tests/cli/exclude.rs index dee35673..d30612eb 100644 --- a/tests/cli/exclude.rs +++ b/tests/cli/exclude.rs @@ -189,8 +189,8 @@ fn restore_exclude_excludes_subtrees() { .assert() .success() .stdout(indoc! { " - / - /hello + + / + + /hello "}) .stderr(""); dest.child("subdir").assert(predicate::path::missing()); diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 801ec1ff..8f04fb86 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -13,13 +13,17 @@ //! Run conserve CLI as a subprocess and test it. +use std::fs::read_to_string; use std::path::PathBuf; use std::process::Command; use assert_cmd::prelude::*; use assert_fs::prelude::*; +use assert_fs::NamedTempFile; use assert_fs::TempDir; +use indoc::indoc; use predicates::prelude::*; +use serde_json::Deserializer; use url::Url; use conserve::test_fixtures::{ScratchArchive, TreeFixture}; @@ -34,6 +38,7 @@ mod versions; #[cfg(unix)] mod unix { + mod diff; mod permissions; } @@ -214,6 +219,7 @@ fn basic_backup() { // TODO: Factor out comparison to expected tree. let restore_dir = TempDir::new().unwrap(); + let restore_json = NamedTempFile::new("restore.json").unwrap(); // Also try --no-progress here; should make no difference because these tests run // without a pty. @@ -221,18 +227,20 @@ fn basic_backup() { .arg("restore") .arg("-v") .arg("--no-progress") + .arg("--no-stats") .arg(&arch_dir) .arg(restore_dir.path()) + .arg("--changes-json") + .arg(restore_json.path()) .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with( - "/\n\ - /hello\n\ - /subdir\n\ - /subdir/subfile\n\ - ", - )); + .stdout(indoc! { " + + / + + /hello + + /subdir + + /subdir/subfile + " }); restore_dir .child("subdir") @@ -246,6 +254,19 @@ fn basic_backup() { .child("subfile") .assert("I like Rust\n"); + let json = read_to_string(restore_json.path()).unwrap(); + dbg!(&json); + let changes: Vec = Deserializer::from_str(&json) + .into_iter::() + .map(Result::unwrap) + .collect(); + dbg!(&changes); + assert_eq!(changes.len(), 4); + assert_eq!(changes[0]["apath"], "/"); + assert_eq!(changes[1]["apath"], "/hello"); + assert_eq!(changes[2]["apath"], "/subdir"); + assert_eq!(changes[3]["apath"], "/subdir/subfile"); + // Try to restore again over the same directory: should decline. run_conserve() .arg("restore") diff --git a/tests/cli/unix/diff.rs b/tests/cli/unix/diff.rs new file mode 100644 index 00000000..29ef4c6a --- /dev/null +++ b/tests/cli/unix/diff.rs @@ -0,0 +1,87 @@ +// Conserve backup system. +// Copyright 2021-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Test `conserve diff` on Unix with symlinks. + +use std::fs; + +use assert_cmd::prelude::*; +use predicates::prelude::*; + +use conserve::test_fixtures::{ScratchArchive, TreeFixture}; + +use crate::run_conserve; + +fn setup_symlink() -> (ScratchArchive, TreeFixture) { + let af = ScratchArchive::new(); + let tf = TreeFixture::new(); + tf.create_dir("subdir"); + tf.create_symlink("subdir/link", "target"); + run_conserve() + .arg("backup") + .arg(af.path()) + .arg(tf.path()) + .assert() + .success(); + (af, tf) +} + +#[test] +pub fn symlink_unchanged() { + let (af, tf) = setup_symlink(); + + run_conserve() + .arg("diff") + .arg(af.path()) + .arg(tf.path()) + .assert() + .success() + .stdout("") + .stderr(predicate::str::is_empty()); + + run_conserve() + .arg("diff") + .arg("--include-unchanged") + .arg(af.path()) + .arg(tf.path()) + .assert() + .success() + .stdout(". /\n. /subdir\n. /subdir/link\n") + .stderr(predicate::str::is_empty()); +} + +#[test] +pub fn symlink_changed() { + let (af, tf) = setup_symlink(); + fs::remove_file(tf.path().join("subdir/link")).unwrap(); + tf.create_symlink("subdir/link", "newtarget"); + + run_conserve() + .arg("diff") + .arg(af.path()) + .arg(tf.path()) + .assert() + .success() + .stdout("* /subdir/link\n") + .stderr(predicate::str::is_empty()); + + run_conserve() + .arg("diff") + .arg("--include-unchanged") + .arg(af.path()) + .arg(tf.path()) + .assert() + .success() + .stdout(". /\n. /subdir\n* /subdir/link\n") + .stderr(predicate::str::is_empty()); +} diff --git a/tests/cli/unix/permissions.rs b/tests/cli/unix/permissions.rs index 0efe5e30..e8f26e9b 100644 --- a/tests/cli/unix/permissions.rs +++ b/tests/cli/unix/permissions.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use assert_cmd::prelude::*; use assert_fs::prelude::*; use assert_fs::TempDir; -use indoc::indoc; +use indoc::{formatdoc, indoc}; use predicates::prelude::*; use crate::run_conserve; @@ -97,12 +97,12 @@ fn backup_unix_permissions() { .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::diff(format!( - "rwxr-xr-x {user:<10} {group:<10} /\n\ - r--r--r-- {user:<10} {group:<10} /hello\n\ - rwxrwxr-x {user:<10} {group:<10} /subdir\n\ - rwxr-xr-x {user:<10} {group:<10} /subdir/subfile\n" - ))); + .stdout(predicate::str::diff(formatdoc! { " + rwxr-xr-x {user:<10} {group:<10} / + r--r--r-- {user:<10} {group:<10} /hello + rwxrwxr-x {user:<10} {group:<10} /subdir + rwxr-xr-x {user:<10} {group:<10} /subdir/subfile + " })); // create a directory to restore to let restore_dir = TempDir::new().unwrap(); @@ -115,13 +115,12 @@ fn backup_unix_permissions() { .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::diff(format!( - "rwxr-xr-x {user:<10} {group:<10} /\n\ - r--r--r-- {user:<10} {group:<10} /hello\n\ - rwxrwxr-x {user:<10} {group:<10} /subdir\n\ - rwxr-xr-x {user:<10} {group:<10} /subdir/subfile\n\ - " - ))); + .stdout(predicate::str::diff(formatdoc! {" + + rwxr-xr-x {user:<10} {group:<10} / + + r--r--r-- {user:<10} {group:<10} /hello + + rwxrwxr-x {user:<10} {group:<10} /subdir + + rwxr-xr-x {user:<10} {group:<10} /subdir/subfile + "})); } #[test] @@ -202,17 +201,18 @@ fn backup_user_and_permissions() { // restore run_conserve() - .args(["restore", "-v", "-l", "--no-progress"]) + .args(["restore", "-v", "-l", "--no-progress", "--no-stats"]) .arg(&arch_dir) .arg(restore_dir.path()) .assert() .success() .stderr(predicate::str::is_empty()) - .stdout(predicate::str::starts_with(format!( - "{} {} /\n\ - {} {} /hello\n\ - {} {} /subdir\n\ - {} {} /subdir/subfile\n\ + .stdout(predicate::str::diff(formatdoc!( + " + + {} {} / + + {} {} /hello + + {} {} /subdir + + {} {} /subdir/subfile ", UnixMode::from(mdata_root.permissions()), Owner::from(&mdata_root), diff --git a/tests/expensive/changes.rs b/tests/expensive/changes.rs index 6ec9449c..02aaf1d3 100644 --- a/tests/expensive/changes.rs +++ b/tests/expensive/changes.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2022 Martin Pool. +// Copyright 2022-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by From e715e207659517e9de36ab5c293ce23074186976 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 11 May 2023 07:09:49 -0700 Subject: [PATCH 20/86] Add codespell config --- .codespell.words | 2 ++ .codespellrc | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 .codespell.words create mode 100644 .codespellrc diff --git a/.codespell.words b/.codespell.words new file mode 100644 index 00000000..d84f1c41 --- /dev/null +++ b/.codespell.words @@ -0,0 +1,2 @@ +crate +ser diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..29dadde5 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +ignore-words = .codespell.words +skip = target,.git From d7a3d50ad38d4fdd8a7f1d2bfcc074ff82c33ca2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 11 May 2023 07:09:57 -0700 Subject: [PATCH 21/86] Fix some typos in docs and comments --- doc/design.md | 4 ++-- doc/manifesto.md | 27 +++++++++++-------------- doc/style.md | 4 ++-- doc/unimplemented/encryption.md | 36 ++++++++++++++++----------------- tests/api/backup.rs | 2 +- tests/api/restore.rs | 2 +- tests/cli/unix/permissions.rs | 2 +- 7 files changed, 36 insertions(+), 41 deletions(-) diff --git a/doc/design.md b/doc/design.md index a6e18107..d9f42947 100644 --- a/doc/design.md +++ b/doc/design.md @@ -98,7 +98,7 @@ the filesystem behavior, they should notice the band has already been created, and abort. Index blocks are written by atomically renaming them in to place. If the block -already exists, the new version (with identical contents) is simpy discarded. +already exists, the new version (with identical contents) is simply discarded. So, concurrent writes of blocks are safe, and indeed can happen from multiple threads in the same process. @@ -196,7 +196,7 @@ well as to the terminal, and at a different level of detail. This implies: - Since the terminal UI is a log target, it must be constructed just once near program startup, and therefore it cannot be on during in-process tests. -Progess bars are drawn only for the small number of main loops that are expected +Progress bars are drawn only for the small number of main loops that are expected to take a long time, and don't implicitly pop up due to IO. ## Diff diff --git a/doc/manifesto.md b/doc/manifesto.md index c0a7f948..58c30411 100644 --- a/doc/manifesto.md +++ b/doc/manifesto.md @@ -96,32 +96,32 @@ touch or rewrite any files other than those being deleted. Storage to cloud object stores, local disks, and removable media are all important. Conserve should rely on only features common across all of them. -- You can write whole files, but not update in place. +* You can write whole files, but not update in place. -- May have relatively long per-file latency on both read and write. +* May have relatively long per-file latency on both read and write. -- Storage bandwidth may be relatively limited relative to the source tree +* Storage bandwidth may be relatively limited relative to the source tree size. -- No filesystem metadata (ownership etc) can be stored directly; it must +* No filesystem metadata (ownership etc) can be stored directly; it must be encoded -- You can list directories (or, "list files starting with a certain prefix") +* You can list directories (or, "list files starting with a certain prefix") -- May or may not be case sensitive. +* May or may not be case sensitive. -- Can't detect whether an empty directory exists or not, and might not have a +* Can't detect whether an empty directory exists or not, and might not have a strong concept of directories, perhaps only ordered names. -- Do not assume that renaming over an existing file is allowed or disallowed. +* Do not assume that renaming over an existing file is allowed or disallowed. -- Conserve can cache information onto the source machine's local disk, but of +* Conserve can cache information onto the source machine's local disk, but of course this cache may be lost or may need to be disabled. (We don't currently do this, and it would keep things simpler and more robust not to.) -- Connection may be lost and the backup terminated at any point. +* Connection may be lost and the backup terminated at any point. -- No guarantee of read-after-write consistency. (In practice, perhaps several +* No guarantee of read-after-write consistency. (In practice, perhaps several seconds after writing the change will be visible.) We cannot assume a remote smart server: the only network calls are @@ -181,7 +181,7 @@ archive. It's very possible that the size of the source relative to the IO bandwidth of the destination means writing all the new data will take hours. This can most -easily happen on the first backup, but als on incremental backups. +easily happen on the first backup, but also on incremental backups. In that case the backup may be interrupted - by the user interrupting it, machine going to sleep, or losing connectivity, or rebooting. @@ -208,7 +208,6 @@ This excludes a few design options taken by other programs: Remembering a save-point on the source machine seems more dangerous than looking in the archive to see what's been stored. - ## Validation Test restores of the whole tree take a long time and users don't do them @@ -224,14 +223,12 @@ to the source directory, to catch corruption or Conserve bugs. These can flag false-positive if there have been intended changes to the source directory after the backup, so the results need to be understandable. - ## Hands-off Conserve will let you set up cron jobs to do daily backups, verification, and retrenchment, and it should then run hands off and entirely unattended. (Users should also do a black-box restore test, which should never fail.) - ## UI Conserve will have a human oriented text UI, and a machine UI that can diff --git a/doc/style.md b/doc/style.md index 776e8343..a1c114d5 100644 --- a/doc/style.md +++ b/doc/style.md @@ -69,7 +69,7 @@ Code in Conserve can be tested in any of three ways: 1. Key features and behaviors accessible through the command-line interface should be tested in `tests/cli`, which runs the `conserve` binary as a - subprocess and examines its output. Since Conserve is + subprocess and examines its output. Since Conserve is primarily intended for use as a command-line tool these are the most important tests to add. @@ -97,7 +97,7 @@ tree) won't have deterministic permissions or mtimes. Use `use crate::xyz` rather than `use super::xyz` to import other things from the Conserve implementation. (Either is valid and they seem just as good, but -let's pick `crate` to be consisent.) +let's pick `crate` to be consistent.) Conserve implementation code and integration tests can say `use crate::*` to include every re-exported symbol, although this isn't recommended for external diff --git a/doc/unimplemented/encryption.md b/doc/unimplemented/encryption.md index 0234394f..16d1476a 100644 --- a/doc/unimplemented/encryption.md +++ b/doc/unimplemented/encryption.md @@ -21,7 +21,7 @@ But we assume that attackers can: - For example, because they control the server to which encrypted archives are written - Or because they can gain physical access to a drive holding backups - See how the encrypted archive changes over time - - For example, if they control the server hosting archives they can collect a trace of + - For example, if they control the server hosting archives they can collect a trace of file reads and writes - Tamper with the encrypted archive content: - Including deleting and modifying files @@ -66,17 +66,17 @@ some separate stable storage, a record of when backups were made. An attacker should not be able to: -- Silently corrupt, change, or remove data, other than reverting to a previous version of the +- Silently corrupt, change, or remove data, other than reverting to a previous version of the archive, including: - Copying files to different names - Removing some files from the tree (other than by reverting to a moment when the backup was incomplete) -- Prevent a machine making a new correct backup. In other words, after +- Prevent a machine making a new correct backup. In other words, after tampering, newly written files in a new backup will still be correct. -- Execute downgrade attacks that manipulate the backup program into +- Execute downgrade attacks that manipulate the backup program into writing unencrypted content or using an attacker-influenced key. -For performance reasons Conserve does not throughly validate all existing +For performance reasons Conserve does not thoroughly validate all existing blocks when it writes a new archive, so corruption of existing blocks may go unnoticed for some time. However `conserve validate` should detect this corruption. @@ -87,7 +87,7 @@ The key can be stored in a file for noninteractive scheduled backups. The key can optionally be stored in some kind of system keyring, so that it is somewhat harder to steal, e.g. so that it is only unlocked when the user is logged in. (At the price of only being available to make backups when the user is logged in, in that case.) -It's important that users keep a copy of the key in a place where it will not be lost if the backup source is lost, e.g. typically not on the same machine. The key should be concisely representable as text. These backups of the key must also be stored somewhere that the user feels is significantly less likely to be compromised than the backup storage itself, otherwise the encrytion is adding no value. +It's important that users keep a copy of the key in a place where it will not be lost if the backup source is lost, e.g. typically not on the same machine. The key should be concisely representable as text. These backups of the key must also be stored somewhere that the user feels is significantly less likely to be compromised than the backup storage itself, otherwise the encryption is adding no value. Test restores or validation should allow the user to try presenting the key as if they were doing a recovery, e.g. by typing it in or using a non-default file, even if it is normally read from a file or keyring. @@ -164,7 +164,7 @@ When the keys are rotated, existing blocks in unchanged files can still match ag ### Block encryption To write a block, it is first hashed, with the hash key. If the hash is already present, that's -enough, and the keyed hash can be used to refer to the block content from the index or +enough, and the keyed hash can be used to refer to the block content from the index or meta-index. Otherwise, the block content is encrypted. (In unencrypted archives the block would be compressed at this point; in encrypted archives it is not.) @@ -217,8 +217,8 @@ Since Tink generates a random IV for each block, IVs are never reused. By the same logic as for Eve, Mallory cannot decrypt block content. -If Mallory blindly changes the content of a block file or truncates it, then -when decrypted it will be discovered to have the wrong keyed hash, which +If Mallory blindly changes the content of a block file or truncates it, then +when decrypted it will be discovered to have the wrong keyed hash, which will be detected as corruption. If Mallory copies one block file in place of another the IV will be wrong, so @@ -236,11 +236,11 @@ It is important that the backup client must not trust the archive's assertion wh ### Assessment: chosen-plaintext attacks -An attacker who can both inject chosen plaintext and observe writes to the archive +An attacker who can both inject chosen plaintext and observe writes to the archive may be able to determine whether the plaintext is already present in the archive. For example, if the attacker injects a 1MB file (which will be written as a single block) and observes that no new large blocks are written, then they can infer -that an identical block was already present at some point in the archive. +that an identical block was already present at some point in the archive. (It does not necessarily prove that the content is present in the most recent tree, only that the block was still present.) @@ -257,13 +257,13 @@ fairly large and hold multiple files, but in some cases Conserve will emit only small data blocks, most obviously when only one small file has changed, but also when changes have to be flushed out to finalize an index block. -The most favorable case for an attacker is if they're trying to guess whether +The most favorable case for an attacker is if they're trying to guess whether a particular single-byte file is present, and they can inject new single-byte files into an otherwise-quiescent archive. The simplest attack is to guess one file at a time, in which case they will likely find the answer after 255 guesses. -Potentially the attacker could make multiple guesses per backup cycle, but -they then face the risk that their small files will be combined into a single -larger block, yielding inconclusive results. +Potentially the attacker could make multiple guesses per backup cycle, but +they then face the risk that their small files will be combined into a single +larger block, yielding inconclusive results. Interestingly, this attack can only be done once per archive, since after each byte is guessed it will then be present in the blockdir and future guesses will @@ -277,10 +277,8 @@ If, as is planned, small files are stored inline in the index then this attack becomes infeasible for any file small enough to make guessing even remotely feasible. -If blocks were compressed, it might be possible for an attacker to inject a -series of chosen plaintexts and gradually measure whether they compress well +If blocks were compressed, it might be possible for an attacker to inject a +series of chosen plaintexts and gradually measure whether they compress well against other files nearby in the tree. Because compression is disabled in encrypted archives the attacker is limited to guessing at whether whole blocks are present, which seems much less tractable. - - diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 7471ff31..4ceaab4f 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -10,7 +10,7 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -//! Tests focussed on backup behavior. +//! Tests focused on backup behavior. use assert_fs::prelude::*; use assert_fs::TempDir; diff --git a/tests/api/restore.rs b/tests/api/restore.rs index fd99f146..f80dc67c 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -10,7 +10,7 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -//! Tests focussed on restore. +//! Tests focused on restore. use std::cell::RefCell; #[cfg(unix)] diff --git a/tests/cli/unix/permissions.rs b/tests/cli/unix/permissions.rs index e8f26e9b..dad8e9b8 100644 --- a/tests/cli/unix/permissions.rs +++ b/tests/cli/unix/permissions.rs @@ -238,7 +238,7 @@ fn backup_user_and_permissions() { } #[test] -/// List an archive with particular encoded permissions, from the first version tha tracked +/// List an archive with particular encoded permissions, from the first version that tracked /// ownership and permissions. /// /// This should succeed even, and especially, if the machine running the tests does From f107b5215cad8e0597bf63b08f2c2d9b6923f497 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 08:17:43 -0700 Subject: [PATCH 22/86] Add codespell dict --- .codespell.dict | 1 + .codespellrc | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 .codespell.dict diff --git a/.codespell.dict b/.codespell.dict new file mode 100644 index 00000000..764c1698 --- /dev/null +++ b/.codespell.dict @@ -0,0 +1 @@ +assertino->assertion diff --git a/.codespellrc b/.codespellrc index 29dadde5..40b9ae00 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,5 @@ [codespell] ignore-words = .codespell.words skip = target,.git +builtin = clear,rare,code +dictionary = .codespell.dict From dbe321228a4ba4248cb440341ec1a341609d1c12 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 09:16:15 -0700 Subject: [PATCH 23/86] backup takes a path for the source --- src/backup.rs | 8 +++++--- src/bin/conserve.rs | 1 - src/gc_lock.rs | 4 ++-- src/stitch.rs | 3 +-- src/stored_file.rs | 2 +- src/test_fixtures.rs | 4 ++-- tests/api/backup.rs | 42 ++++++++++++++++++-------------------- tests/api/diff.rs | 2 +- tests/api/gc.rs | 6 +++--- tests/api/old_archives.rs | 2 +- tests/api/restore.rs | 2 +- tests/expensive/changes.rs | 2 +- 12 files changed, 38 insertions(+), 40 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index c51208eb..a4081cd8 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -17,6 +17,7 @@ use std::convert::TryInto; use std::fmt; use std::io::prelude::*; +use std::path::Path; use std::time::{Duration, Instant}; use derive_more::{Add, AddAssign}; @@ -72,23 +73,24 @@ impl Default for BackupOptions<'_> { /// Returns statistics about what was copied. pub fn backup( archive: &Archive, - source: &LiveTree, + source_path: &Path, options: &BackupOptions, ) -> Result { let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; let mut stats = BackupStats::default(); let bar = Bar::new(); + let source_tree = LiveTree::open(source_path)?; let mut scanned_file_bytes = 0; let mut entries_new = 0; let mut entries_changed = 0; let mut entries_unchanged = 0; - let entry_iter = source.iter_entries(Apath::root(), options.exclude.clone())?; + let entry_iter = source_tree.iter_entries(Apath::root(), options.exclude.clone())?; for entry_group in entry_iter.chunks(options.max_entries_per_hunk).into_iter() { for entry in entry_group { - match writer.copy_entry(&entry, source) { + match writer.copy_entry(&entry, &source_tree) { Err(err) => { error!(?entry, ?err, "Error copying entry to backup"); stats.errors += 1; diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index ffa37cae..3c8a2499 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -299,7 +299,6 @@ impl Command { source, verbose, } => { - let source = &LiveTree::open(source)?; let options = BackupOptions { exclude: Exclude::from_patterns_and_files(exclude, exclude_from)?, change_callback: make_change_callback( diff --git a/src/gc_lock.rs b/src/gc_lock.rs index 2b2f4501..ee96cdde 100644 --- a/src/gc_lock.rs +++ b/src/gc_lock.rs @@ -126,7 +126,7 @@ mod test { fn completed_backup_ok() { let archive = ScratchArchive::new(); let source = TreeFixture::new(); - backup(&archive, &source.live_tree(), &BackupOptions::default()).unwrap(); + backup(&archive, source.path(), &BackupOptions::default()).unwrap(); let delete_guard = GarbageCollectionLock::new(&archive).unwrap(); delete_guard.check().unwrap(); } @@ -136,7 +136,7 @@ mod test { let archive = ScratchArchive::new(); let source = TreeFixture::new(); let _delete_guard = GarbageCollectionLock::new(&archive).unwrap(); - let backup_result = backup(&archive, &source.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, source.path(), &BackupOptions::default()); assert_eq!( backup_result.expect_err("backup fails").to_string(), "Archive is locked for garbage collection" diff --git a/src/stitch.rs b/src/stitch.rs index 20aa02ad..f6c8d551 100644 --- a/src/stitch.rs +++ b/src/stitch.rs @@ -267,9 +267,8 @@ mod test { let tf = TreeFixture::new(); tf.create_file("file_a"); - let lt = tf.live_tree(); let af = ScratchArchive::new(); - backup(&af, <, &BackupOptions::default()).expect("backup should work"); + backup(&af, tf.path(), &BackupOptions::default()).expect("backup should work"); af.transport().remove_file("b0000/BANDTAIL").unwrap(); let band_ids = af.list_band_ids().expect("should list bands"); diff --git a/src/stored_file.rs b/src/stored_file.rs index dc2714e9..2127724b 100644 --- a/src/stored_file.rs +++ b/src/stored_file.rs @@ -10,7 +10,7 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -///! Access a file stored in the archive. +//! Access a file stored in the archive. use crate::stats::Sizes; use crate::*; diff --git a/src/test_fixtures.rs b/src/test_fixtures.rs index 4f792462..e8a2c18e 100644 --- a/src/test_fixtures.rs +++ b/src/test_fixtures.rs @@ -65,10 +65,10 @@ impl ScratchArchive { } let options = &BackupOptions::default(); - backup(&self.archive, &srcdir.live_tree(), options).unwrap(); + backup(&self.archive, srcdir.path(), options).unwrap(); srcdir.create_file("hello2"); - backup(&self.archive, &srcdir.live_tree(), options).unwrap(); + backup(&self.archive, srcdir.path(), options).unwrap(); } pub fn transport(&self) -> &dyn Transport { diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 4ceaab4f..4dbfd925 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -32,7 +32,7 @@ pub fn simple_backup() { let srcdir = TreeFixture::new(); srcdir.create_file("hello"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let copy_stats = backup(&af, srcdir.path(), &BackupOptions::default()).expect("backup"); assert_eq!(copy_stats.index_builder_stats.index_hunks, 1); assert_eq!(copy_stats.files, 1); assert_eq!(copy_stats.deduplicated_blocks, 0); @@ -64,12 +64,11 @@ pub fn simple_backup_with_excludes() -> Result<()> { srcdir.create_file("baz"); // TODO: Include a symlink only on Unix. let exclude = Exclude::from_strings(["/**/baz", "/**/bar", "/**/fooo*"]).unwrap(); - let source = srcdir.live_tree(); let options = BackupOptions { exclude, ..BackupOptions::default() }; - let copy_stats = backup(&af, &source, &options).expect("backup"); + let copy_stats = backup(&af, srcdir.path(), &options).expect("backup"); check_backup(&af); @@ -120,12 +119,11 @@ pub fn backup_more_excludes() { srcdir.create_file("bar"); let exclude = Exclude::from_strings(["/**/foo*", "/**/baz"]).unwrap(); - let source = srcdir.live_tree(); let options = BackupOptions { exclude, ..Default::default() }; - let stats = backup(&af, &source, &options).expect("backup"); + let stats = backup(&af, srcdir.path(), &options).expect("backup"); assert_eq!(1, stats.written_blocks); assert_eq!(1, stats.files); @@ -187,7 +185,7 @@ fn large_file() { let large_content = vec![b'a'; 4 << 20]; tf.create_file_with_contents("large", &large_content); - let backup_stats = backup(&af, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let backup_stats = backup(&af, tf.path(), &BackupOptions::default()).expect("backup"); assert_eq!(backup_stats.new_files, 1); // First 1MB should be new; remainder should be deduplicated. assert_eq!(backup_stats.uncompressed_bytes, 1 << 20); @@ -221,7 +219,7 @@ fn source_unreadable() { tf.make_file_unreadable("b_unreadable"); - let stats = backup(&af, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, tf.path(), &BackupOptions::default()).expect("backup"); assert_eq!(stats.errors, 1); assert_eq!(stats.new_files, 3); assert_eq!(stats.files, 3); @@ -251,7 +249,7 @@ fn mtime_before_epoch() { assert_eq!(entries[1].apath(), "/old_file"); let af = ScratchArchive::new(); - backup(&af, &tf.live_tree(), &BackupOptions::default()) + backup(&af, tf.path(), &BackupOptions::default()) .expect("backup shouldn't crash on before-epoch mtimes"); } @@ -261,7 +259,7 @@ pub fn symlink() { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_symlink("symlink", "/a/broken/destination"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let copy_stats = backup(&af, srcdir.path(), &BackupOptions::default()).expect("backup"); assert_eq!(0, copy_stats.files); assert_eq!(1, copy_stats.symlinks); @@ -290,7 +288,7 @@ pub fn empty_file_uses_zero_blocks() { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_file_with_contents("empty", &[]); - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats = backup(&af, srcdir.path(), &BackupOptions::default()).unwrap(); assert_eq!(1, stats.files); assert_eq!(stats.written_blocks, 0); @@ -322,7 +320,7 @@ pub fn detect_unmodified() { srcdir.create_file("bbb"); let options = BackupOptions::default(); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, srcdir.path(), &options).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 2); @@ -330,7 +328,7 @@ pub fn detect_unmodified() { // Make a second backup from the same tree, and we should see that // both files are unmodified. - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, srcdir.path(), &options).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 0); @@ -340,7 +338,7 @@ pub fn detect_unmodified() { // as unmodified. srcdir.create_file_with_contents("bbb", b"longer content for bbb"); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, srcdir.path(), &options).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 0); @@ -356,7 +354,7 @@ pub fn detect_minimal_mtime_change() { srcdir.create_file_with_contents("bbb", b"longer content for bbb"); let options = BackupOptions::default(); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, srcdir.path(), &options).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 2); @@ -378,7 +376,7 @@ pub fn detect_minimal_mtime_change() { } } - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, srcdir.path(), &options).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.unmodified_files, 1); } @@ -390,7 +388,7 @@ fn small_files_combined_two_backups() { srcdir.create_file("file1"); srcdir.create_file("file2"); - let stats1 = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats1 = backup(&af, srcdir.path(), &BackupOptions::default()).unwrap(); // Although the two files have the same content, we do not yet dedupe them // within a combined block, so the block is different to when one identical // file is stored alone. This could be fixed. @@ -402,7 +400,7 @@ fn small_files_combined_two_backups() { // Add one more file, also identical, but it is not combined with the previous blocks. // This is a shortcoming of the current dedupe approach. srcdir.create_file("file3"); - let stats2 = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats2 = backup(&af, srcdir.path(), &BackupOptions::default()).unwrap(); assert_eq!(stats2.new_files, 1); assert_eq!(stats2.unmodified_files, 2); assert_eq!(stats2.written_blocks, 1); @@ -424,7 +422,7 @@ fn many_small_files_combined_to_one_block() { format!("something about {i}").as_bytes(), ); } - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, srcdir.path(), &BackupOptions::default()).expect("backup"); assert_eq!( stats.index_builder_stats.index_hunks, 2, "expect exactly 2 hunks" @@ -470,7 +468,7 @@ pub fn mixed_medium_small_files_two_hunks() { srcdir.create_file(&name); } } - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, srcdir.path(), &BackupOptions::default()).expect("backup"); assert_eq!( stats.index_builder_stats.index_hunks, 2, "expect exactly 2 hunks" @@ -511,7 +509,7 @@ fn detect_unchanged_from_stitched_index() { // Use small hunks for easier manipulation. let stats = backup( &af, - &srcdir.live_tree(), + srcdir.path(), &BackupOptions { max_entries_per_hunk: 1, ..Default::default() @@ -526,7 +524,7 @@ fn detect_unchanged_from_stitched_index() { srcdir.create_file_with_contents("a", b"new a contents"); let stats = backup( &af, - &srcdir.live_tree(), + srcdir.path(), &BackupOptions { max_entries_per_hunk: 1, ..Default::default() @@ -547,7 +545,7 @@ fn detect_unchanged_from_stitched_index() { // index from both b0 and b1. let stats = backup( &af, - &srcdir.live_tree(), + srcdir.path(), &BackupOptions { max_entries_per_hunk: 1, ..Default::default() diff --git a/tests/api/diff.rs b/tests/api/diff.rs index 52865461..7ed7679a 100644 --- a/tests/api/diff.rs +++ b/tests/api/diff.rs @@ -23,7 +23,7 @@ fn diff_unchanged() { tf.create_file_with_contents("thing", b"contents of thing"); let lt = tf.live_tree(); - let stats = backup(&a, <, &BackupOptions::default()).unwrap(); + let stats = backup(&a, tf.path(), &BackupOptions::default()).unwrap(); assert_eq!(stats.new_files, 1); let st = a.open_stored_tree(BandSelectionPolicy::Latest).unwrap(); diff --git a/tests/api/gc.rs b/tests/api/gc.rs index 86295285..3a59940e 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -26,7 +26,7 @@ fn unreferenced_blocks() { .parse() .unwrap(); - let _copy_stats = backup(&archive, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let _copy_stats = backup(&archive, tf.path(), &BackupOptions::default()).expect("backup"); // Delete the band and index std::fs::remove_dir_all(archive.path().join("b0000")).unwrap(); @@ -98,7 +98,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { let lock1 = GarbageCollectionLock::new(&archive)?; // Backup should fail while gc lock is held. - let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, tf.path(), &BackupOptions::default()); match backup_result { Err(Error::GarbageCollectionLockHeld) => (), other => panic!("unexpected result {other:?}"), @@ -115,7 +115,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { )?; // Backup should now succeed. - let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, tf.path(), &BackupOptions::default()); assert!(backup_result.is_ok()); Ok(()) diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index 07d961e6..f908a0cd 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -212,7 +212,7 @@ fn restore_modify_backup() { let emitted = RefCell::new(Vec::new()); let backup_stats = backup( &new_archive, - &LiveTree::open(working_tree.path()).unwrap(), + working_tree.path(), &BackupOptions { change_callback: Some(Box::new(|change| { emitted diff --git a/tests/api/restore.rs b/tests/api/restore.rs index f80dc67c..7282742f 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -149,7 +149,7 @@ fn restore_symlink() { let years_ago = FileTime::from_unix_time(189216000, 0); set_symlink_file_times(srcdir.path().join("symlink"), years_ago, years_ago).unwrap(); - backup(&af, &srcdir.live_tree(), &Default::default()).unwrap(); + backup(&af, srcdir.path(), &Default::default()).unwrap(); let restore_dir = TempDir::new().unwrap(); restore(&af, restore_dir.path(), &Default::default()).unwrap(); diff --git a/tests/expensive/changes.rs b/tests/expensive/changes.rs index 02aaf1d3..8e8be869 100644 --- a/tests/expensive/changes.rs +++ b/tests/expensive/changes.rs @@ -93,7 +93,7 @@ fn backup_sequential_changes(changes: &[TreeChange]) { max_entries_per_hunk: 3, ..BackupOptions::default() }; - backup(&archive, &tf.live_tree(), &options).unwrap(); + backup(&archive, tf.path(), &options).unwrap(); let snapshot = TempDir::new().unwrap(); cp_r::CopyOptions::default() .copy_tree(tf.path(), snapshot.path()) From 7ad33af6dc43c7fb7229124dde9d0f22b0972f66 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 09:05:49 -0700 Subject: [PATCH 24/86] WIP: test damaged archives Towards #210 --- src/backup.rs | 1 + tests/damage/README.md | 10 ++++++++++ tests/damage/main.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tests/damage/README.md create mode 100644 tests/damage/main.rs diff --git a/src/backup.rs b/src/backup.rs index a4081cd8..713873aa 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -71,6 +71,7 @@ impl Default for BackupOptions<'_> { /// Backup a source directory into a new band in the archive. /// /// Returns statistics about what was copied. +// TODO: Maybe this should take a Path and the LiveTree should be an implementation detail? pub fn backup( archive: &Archive, source_path: &Path, diff --git a/tests/damage/README.md b/tests/damage/README.md new file mode 100644 index 00000000..fdc9180c --- /dev/null +++ b/tests/damage/README.md @@ -0,0 +1,10 @@ +# damage tests + +Conserve tries to still allow the archive to be read, and future backups to be written, +even if some files are damaged: truncated, corrupt, missing, or unreadable. + +This is not yet achieved in every case, but the format and code are designed to +work towards this goal. + +These API tests write an archive, create some damage, and then try to read other +information, write future backups, and validate. diff --git a/tests/damage/main.rs b/tests/damage/main.rs new file mode 100644 index 00000000..05369bdb --- /dev/null +++ b/tests/damage/main.rs @@ -0,0 +1,32 @@ +// Conserve backup system. +// Copyright 2020-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +use assert_fs::TempDir; + +use assert_fs::prelude::*; +use conserve::backup; +use conserve::Archive; + +#[test] +fn truncated_band_head() { + let mut archive_dir = TempDir::new().unwrap(); + let mut source_dir = TempDir::new().unwrap(); + + let mut archive = Archive::create_path(archive_dir.path()).expect("create archive"); + source_dir + .child("file") + .write_str("content in first backup") + .unwrap(); + + // backup(&mut archive, source_dir.path()); +} From a82178c91da97e399dea41387f089f928b59824d Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 09:31:03 -0700 Subject: [PATCH 25/86] Add failing test for damaged bandhead --- tests/damage/main.rs | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 05369bdb..d8a0e7c7 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -11,16 +11,27 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -use assert_fs::TempDir; +use std::fs::read_to_string; use assert_fs::prelude::*; +use assert_fs::TempDir; +use predicates::prelude::*; + use conserve::backup; use conserve::Archive; +use conserve::BackupOptions; +use tracing_test::traced_test; +// TODO: Also test other files. +// TODO: Also test other types of damage, including missing files, +// permission denied (as a kind of IOError), and binary junk. + +#[traced_test] #[test] +#[should_panic(expected = "Failed to open band: DeserializeJson")] // TODO: Should pass! fn truncated_band_head() { - let mut archive_dir = TempDir::new().unwrap(); - let mut source_dir = TempDir::new().unwrap(); + let archive_dir = TempDir::new().unwrap(); + let source_dir = TempDir::new().unwrap(); let mut archive = Archive::create_path(archive_dir.path()).expect("create archive"); source_dir @@ -28,5 +39,23 @@ fn truncated_band_head() { .write_str("content in first backup") .unwrap(); - // backup(&mut archive, source_dir.path()); + let backup_options = BackupOptions::default(); + backup(&mut archive, source_dir.path(), &backup_options).expect("initial backup"); + + let bandhead = archive_dir.child("b0000").child("BANDHEAD"); + bandhead.assert(predicate::path::exists()); + println!( + "initial bandhead contents: {:?}", + read_to_string(&bandhead).expect("read bandhead") + ); + bandhead.write_str("").expect("truncate bandhead"); + + // A second backup should succeed. + source_dir + .child("file") + .write_str("content in second backup") + .unwrap(); + + backup(&mut archive, source_dir.path(), &backup_options) + .expect("write second backup even though first bandhead is damaged"); } From 3e3a6f0875ab7bf3a92187b79bf8389aa8853682 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 09:42:37 -0700 Subject: [PATCH 26/86] Factor out Damage strategy --- tests/damage/main.rs | 21 ++++++++------------ tests/damage/strategy.rs | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 tests/damage/strategy.rs diff --git a/tests/damage/main.rs b/tests/damage/main.rs index d8a0e7c7..de96d5b7 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -11,17 +11,18 @@ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -use std::fs::read_to_string; - use assert_fs::prelude::*; use assert_fs::TempDir; -use predicates::prelude::*; +// use predicates::prelude::*; use conserve::backup; use conserve::Archive; use conserve::BackupOptions; use tracing_test::traced_test; +mod strategy; +use strategy::Damage; + // TODO: Also test other files. // TODO: Also test other types of damage, including missing files, // permission denied (as a kind of IOError), and binary junk. @@ -33,22 +34,16 @@ fn truncated_band_head() { let archive_dir = TempDir::new().unwrap(); let source_dir = TempDir::new().unwrap(); - let mut archive = Archive::create_path(archive_dir.path()).expect("create archive"); + let archive = Archive::create_path(archive_dir.path()).expect("create archive"); source_dir .child("file") .write_str("content in first backup") .unwrap(); let backup_options = BackupOptions::default(); - backup(&mut archive, source_dir.path(), &backup_options).expect("initial backup"); + backup(&archive, source_dir.path(), &backup_options).expect("initial backup"); - let bandhead = archive_dir.child("b0000").child("BANDHEAD"); - bandhead.assert(predicate::path::exists()); - println!( - "initial bandhead contents: {:?}", - read_to_string(&bandhead).expect("read bandhead") - ); - bandhead.write_str("").expect("truncate bandhead"); + Damage::Truncate.damage(&archive_dir.child("b0000").child("BANDHEAD")); // A second backup should succeed. source_dir @@ -56,6 +51,6 @@ fn truncated_band_head() { .write_str("content in second backup") .unwrap(); - backup(&mut archive, source_dir.path(), &backup_options) + backup(&archive, source_dir.path(), &backup_options) .expect("write second backup even though first bandhead is damaged"); } diff --git a/tests/damage/strategy.rs b/tests/damage/strategy.rs new file mode 100644 index 00000000..08d5465d --- /dev/null +++ b/tests/damage/strategy.rs @@ -0,0 +1,42 @@ +// Conserve backup system. +// Copyright 2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Strategies for damaging files. + +use std::fs::OpenOptions; +use std::path::Path; + +/// A way of damaging a file in an archive. +#[derive(Debug, Clone)] +pub enum Damage { + /// Truncate the file to zero bytes. + Truncate, +} + +impl Damage { + /// Apply this damage to a file. + /// + /// The file must already exist. + pub fn damage(&self, path: &Path) { + match self { + Damage::Truncate => { + assert!(path.exists(), "{path:?} does not exist"); + OpenOptions::new() + .write(true) + .truncate(true) + .open(path) + .expect("truncate file"); + } + } + } +} From d4e67e37850c7fe960b4fa2a269ca0ad747745ad Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 10:36:15 -0700 Subject: [PATCH 27/86] When stitching, skip backups we can't open Fixes #210 --- src/stitch.rs | 19 +++++++++++++------ tests/damage/main.rs | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/stitch.rs b/src/stitch.rs index f6c8d551..eaa8d045 100644 --- a/src/stitch.rs +++ b/src/stitch.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2017, 2018, 2019, 2020 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -28,6 +28,8 @@ //! seen. //! * Bands might be deleted, so their numbers are not contiguous. +use tracing::warn; + use crate::index::IndexEntryIter; use crate::*; @@ -79,6 +81,7 @@ impl Iterator for IterStitchedIndexHunks { fn next(&mut self) -> Option { loop { + // Until we find the next hunk or run out of bands. // If we're already reading an index, and it has more content, return that. if let Some(index_hunks) = &mut self.index_hunks { // An index iterator must be assigned to a band. @@ -106,12 +109,16 @@ impl Iterator for IterStitchedIndexHunks { } if let Some(band_id) = &self.band_id { + let band = match Band::open(&self.archive, band_id) { + Ok(band) => band, + Err(err) => { + warn!(?err, ?band_id, "Failed to open band, skipping it"); + self.band_id = previous_existing_band(&self.archive, band_id); + continue; + } + }; // Start reading this new index and skip forward until after last_apath - let mut iter_hunks = Band::open(&self.archive, band_id) - .expect("Failed to open band") - .index() - .iter_hunks(); - + let mut iter_hunks = band.index().iter_hunks(); if let Some(last) = &self.last_apath { iter_hunks = iter_hunks.advance_to_after(last) } diff --git a/tests/damage/main.rs b/tests/damage/main.rs index de96d5b7..3ffc1527 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -16,8 +16,11 @@ use assert_fs::TempDir; // use predicates::prelude::*; use conserve::backup; +use conserve::restore; use conserve::Archive; use conserve::BackupOptions; +use conserve::RestoreOptions; +use dir_assert::assert_paths; use tracing_test::traced_test; mod strategy; @@ -29,8 +32,7 @@ use strategy::Damage; #[traced_test] #[test] -#[should_panic(expected = "Failed to open band: DeserializeJson")] // TODO: Should pass! -fn truncated_band_head() { +fn backup_after_damage() { let archive_dir = TempDir::new().unwrap(); let source_dir = TempDir::new().unwrap(); @@ -50,7 +52,15 @@ fn truncated_band_head() { .child("file") .write_str("content in second backup") .unwrap(); - backup(&archive, source_dir.path(), &backup_options) .expect("write second backup even though first bandhead is damaged"); + + // Can restore the second backup + let restore_dir = TempDir::new().unwrap(); + restore(&archive, restore_dir.path(), &RestoreOptions::default()) + .expect("restore second backup"); + + // Since the second backup rewrote the single file in the backup (and the root dir), + // we should get all the content back out. + assert_paths!(source_dir.path(), restore_dir.path()); } From 6887651affc463a8f6797f47fe9c6ebc9badca81 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 10:56:45 -0700 Subject: [PATCH 28/86] Add Damage::Delete --- Cargo.lock | 202 +++++++++++++++++++++++++++++++++------ Cargo.toml | 7 +- tests/damage/main.rs | 8 +- tests/damage/strategy.rs | 12 ++- 4 files changed, 192 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 108e5357..987d0ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,8 +188,8 @@ checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", ] @@ -247,6 +247,7 @@ dependencies = [ "rayon", "readahead-iterator", "regex", + "rstest", "semver", "serde", "serde_json", @@ -334,7 +335,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.23", + "quote 1.0.27", "syn 1.0.107", ] @@ -345,8 +346,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "rustc_version", "syn 1.0.107", ] @@ -453,6 +454,101 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2 1.0.58", + "quote 1.0.27", + "syn 2.0.16", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getrandom" version = "0.2.8" @@ -712,8 +808,8 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", ] @@ -966,8 +1062,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", "version_check", ] @@ -978,8 +1074,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "version_check", ] @@ -994,9 +1090,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] @@ -1071,11 +1167,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ - "proc-macro2 1.0.50", + "proc-macro2 1.0.58", ] [[package]] @@ -1208,6 +1304,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "rstest" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290ca1a1c8ca7edb7c3283bd44dc35dd54fdec6253a3912e201ba1072018fca8" +dependencies = [ + "cfg-if", + "proc-macro2 1.0.58", + "quote 1.0.27", + "rustc_version", + "syn 1.0.107", + "unicode-ident", +] + [[package]] name = "rustc_version" version = "0.4.0" @@ -1285,8 +1407,8 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", ] @@ -1316,6 +1438,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -1357,8 +1488,19 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +dependencies = [ + "proc-macro2 1.0.58", + "quote 1.0.27", "unicode-ident", ] @@ -1416,8 +1558,8 @@ version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", ] @@ -1509,8 +1651,8 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", ] @@ -1586,7 +1728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "744324b12d69a9fc1edea4b38b7b1311295b662d161ad5deac17bb1358224a08" dependencies = [ "lazy_static", - "quote 1.0.23", + "quote 1.0.27", "syn 1.0.107", ] @@ -1707,8 +1849,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", "wasm-bindgen-shared", ] @@ -1719,7 +1861,7 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ - "quote 1.0.23", + "quote 1.0.27", "wasm-bindgen-macro-support", ] @@ -1729,8 +1871,8 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ - "proc-macro2 1.0.50", - "quote 1.0.23", + "proc-macro2 1.0.58", + "quote 1.0.27", "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", diff --git a/Cargo.toml b/Cargo.toml index 886c7ae2..d6465ce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,11 @@ snap = "1.0.0" tempfile = "3" thiserror = "1.0.19" thousands = "0.2.0" -time = { version = "0.3", features = ["local-offset", "serde", "serde-human-readable"] } +time = { version = "0.3", features = [ + "local-offset", + "serde", + "serde-human-readable", +] } tracing = "0.1" tracing-appender = "0.2" unix_mode = "0.1" @@ -72,6 +76,7 @@ predicates = "2" pretty_assertions = "1.0" proptest = "1.0" proptest-derive = "0.3" +rstest = { version = "0.17", features = [] } tracing-test = { version = "0.2", features = ["no-env-filter"] } [features] diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 3ffc1527..69ae1605 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -21,18 +21,18 @@ use conserve::Archive; use conserve::BackupOptions; use conserve::RestoreOptions; use dir_assert::assert_paths; +use rstest::rstest; use tracing_test::traced_test; mod strategy; use strategy::Damage; // TODO: Also test other files. -// TODO: Also test other types of damage, including missing files, -// permission denied (as a kind of IOError), and binary junk. +#[rstest] #[traced_test] #[test] -fn backup_after_damage() { +fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damage) { let archive_dir = TempDir::new().unwrap(); let source_dir = TempDir::new().unwrap(); @@ -45,7 +45,7 @@ fn backup_after_damage() { let backup_options = BackupOptions::default(); backup(&archive, source_dir.path(), &backup_options).expect("initial backup"); - Damage::Truncate.damage(&archive_dir.child("b0000").child("BANDHEAD")); + damage.damage(&archive_dir.child("b0000").child("BANDHEAD")); // A second backup should succeed. source_dir diff --git a/tests/damage/strategy.rs b/tests/damage/strategy.rs index 08d5465d..d692ce9e 100644 --- a/tests/damage/strategy.rs +++ b/tests/damage/strategy.rs @@ -13,7 +13,7 @@ //! Strategies for damaging files. -use std::fs::OpenOptions; +use std::fs::{remove_file, OpenOptions}; use std::path::Path; /// A way of damaging a file in an archive. @@ -21,6 +21,11 @@ use std::path::Path; pub enum Damage { /// Truncate the file to zero bytes. Truncate, + + /// Delete the file. + Delete, + // TODO: Also test other types of damage, including + // permission denied (as a kind of IOError), and binary junk. } impl Damage { @@ -28,15 +33,18 @@ impl Damage { /// /// The file must already exist. pub fn damage(&self, path: &Path) { + assert!(path.exists(), "{path:?} does not exist"); match self { Damage::Truncate => { - assert!(path.exists(), "{path:?} does not exist"); OpenOptions::new() .write(true) .truncate(true) .open(path) .expect("truncate file"); } + Damage::Delete => { + remove_file(path).expect("delete file"); + } } } } From 8d3fb6e0bb39f6081e9851bf9ab6bb565cf6eb2f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 10:59:36 -0700 Subject: [PATCH 29/86] Comment --- tests/damage/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 69ae1605..a59fbfaa 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -27,7 +27,8 @@ use tracing_test::traced_test; mod strategy; use strategy::Damage; -// TODO: Also test other files. +// TODO: Also test damage to other files: band tail, index hunks, data blocks, etc. +// TODO: Test that you can delete a damaged backup. #[rstest] #[traced_test] @@ -63,4 +64,7 @@ fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damag // Since the second backup rewrote the single file in the backup (and the root dir), // we should get all the content back out. assert_paths!(source_dir.path(), restore_dir.path()); + + // TODO: List versions. + // TODO: List contents of second backup. } From e9f51aee0841695eb4d34df3d8424ff8d9926f8d Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 11:10:22 -0700 Subject: [PATCH 30/86] Can list versions and tree contents after damage --- tests/damage/main.rs | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index a59fbfaa..b85b5c8c 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -13,16 +13,16 @@ use assert_fs::prelude::*; use assert_fs::TempDir; -// use predicates::prelude::*; - -use conserve::backup; -use conserve::restore; -use conserve::Archive; -use conserve::BackupOptions; -use conserve::RestoreOptions; +use conserve::Exclude; use dir_assert::assert_paths; +use indoc::indoc; +use pretty_assertions::assert_eq; use rstest::rstest; use tracing_test::traced_test; +// use predicates::prelude::*; + +use conserve::show::show_entry_names; +use conserve::{backup, restore, Apath, Archive, BackupOptions, BandId, ReadTree, RestoreOptions}; mod strategy; use strategy::Damage; @@ -65,6 +65,30 @@ fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damag // we should get all the content back out. assert_paths!(source_dir.path(), restore_dir.path()); - // TODO: List versions. - // TODO: List contents of second backup. + // You can see both versions. + let versions = archive.list_band_ids().expect("list versions"); + assert_eq!(versions, [BandId::zero(), BandId::new(&[1])]); + + // Can list the contents of the second backup. + // let mut listing = String::new(); + // TODO: Better API for this! + let mut listing: Vec = Vec::new(); + show_entry_names( + archive + .open_stored_tree(conserve::BandSelectionPolicy::Latest) + .expect("open second backup") + .iter_entries(Apath::root(), Exclude::nothing()) + .expect("list entries"), + &mut listing, + false, + ) + .expect("show entry names"); + + assert_eq!( + String::from_utf8(listing).unwrap(), + indoc! {" + / + /file + "} + ); } From 8ce39baaf6ba2300321ebfb026937af4e68ad9fe Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 11:12:36 -0700 Subject: [PATCH 31/86] Can validate after damage --- tests/damage/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index b85b5c8c..4f554323 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -22,7 +22,10 @@ use tracing_test::traced_test; // use predicates::prelude::*; use conserve::show::show_entry_names; -use conserve::{backup, restore, Apath, Archive, BackupOptions, BandId, ReadTree, RestoreOptions}; +use conserve::{ + backup, restore, Apath, Archive, BackupOptions, BandId, ReadTree, RestoreOptions, + ValidateOptions, +}; mod strategy; use strategy::Damage; @@ -91,4 +94,10 @@ fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damag /file "} ); + + // Validation completes although with warnings. + // TODO: This should return problems that we can inspect. + archive + .validate(&ValidateOptions::default()) + .expect("validate"); } From b8d2431a5f9753848084ba7d560f222514fec591 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 11:12:56 -0700 Subject: [PATCH 32/86] Comment --- tests/damage/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 4f554323..90c4e873 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -31,7 +31,7 @@ mod strategy; use strategy::Damage; // TODO: Also test damage to other files: band tail, index hunks, data blocks, etc. -// TODO: Test that you can delete a damaged backup. +// TODO: Test that you can delete a damaged backup; then there are no problems. #[rstest] #[traced_test] From 064048b14ec86d5b9a038c0e515c5a71a38a0995 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 11:26:18 -0700 Subject: [PATCH 33/86] Turn off rstest async feature --- Cargo.lock | 117 ----------------------------------------------------- Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 987d0ec5..061e0c02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,101 +454,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "getrandom" version = "0.2.8" @@ -1310,8 +1215,6 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" dependencies = [ - "futures", - "futures-timer", "rstest_macros", "rustc_version", ] @@ -1438,15 +1341,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - [[package]] name = "smallvec" version = "1.10.0" @@ -1493,17 +1387,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" -dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "unicode-ident", -] - [[package]] name = "tempfile" version = "3.3.0" diff --git a/Cargo.toml b/Cargo.toml index d6465ce7..15cdc859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ predicates = "2" pretty_assertions = "1.0" proptest = "1.0" proptest-derive = "0.3" -rstest = { version = "0.17", features = [] } +rstest = { version = "0.17", default-features = false } tracing-test = { version = "0.2", features = ["no-env-filter"] } [features] From 196fc05b40d560d1190a843bbcea26b1684fa07c Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 11:27:08 -0700 Subject: [PATCH 34/86] Update clap --- Cargo.lock | 313 ++++++++++++++++++++++++++++++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 240 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 061e0c02..eb7687e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,6 +22,55 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "arrayvec" version = "0.4.12" @@ -166,41 +215,46 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.1.4" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" dependencies = [ - "bitflags", + "clap_builder", "clap_derive", - "clap_lex", - "is-terminal", "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", "strsim", - "termcolor", "terminal_size", ] [[package]] name = "clap_derive" -version = "4.1.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", - "proc-macro-error", "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", ] [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" -dependencies = [ - "os_str_bytes", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "clicolors-control" @@ -214,6 +268,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "conserve" version = "23.2.0" @@ -399,6 +459,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -427,7 +498,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -522,6 +593,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -587,19 +664,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.1", "io-lifetimes", - "rustix", - "windows-sys", + "rustix 0.37.3", + "windows-sys 0.48.0", ] [[package]] @@ -644,6 +721,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "lock_api" version = "0.4.9" @@ -844,12 +927,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - [[package]] name = "output_vt100" version = "0.1.3" @@ -885,7 +962,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -960,30 +1037,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 1.0.107", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "version_check", -] - [[package]] name = "proc-macro2" version = "0.4.30" @@ -1249,11 +1302,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ "bitflags", - "errno", + "errno 0.2.8", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.1.4", + "windows-sys 0.42.0", +] + +[[package]] +name = "rustix" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2" +dependencies = [ + "bitflags", + "errno 0.3.1", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.45.0", ] [[package]] @@ -1387,6 +1454,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +dependencies = [ + "proc-macro2 1.0.58", + "quote 1.0.27", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -1401,23 +1479,14 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" dependencies = [ - "rustix", - "windows-sys", + "rustix 0.36.7", + "windows-sys 0.42.0", ] [[package]] @@ -1669,6 +1738,12 @@ dependencies = [ "log", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "valuable" version = "0.1.0" @@ -1814,13 +1889,61 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -1829,42 +1952,84 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "yansi" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 15cdc859..f1f12f20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ users = "0.11" nix = "0.26" [dependencies.clap] -version = "4.0" +version = "4.3" features = ["derive", "deprecated", "wrap_help"] [dependencies.nutmeg] From d996e4b2ed21c757c8121dbed8db0b5206da3ea6 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 15:32:48 -0700 Subject: [PATCH 35/86] Move API tests out --- src/archive.rs | 108 -------------------------------------- tests/api/archive.rs | 121 +++++++++++++++++++++++++++++++++++++++++++ tests/api/main.rs | 1 + 3 files changed, 122 insertions(+), 108 deletions(-) create mode 100644 tests/api/archive.rs diff --git a/src/archive.rs b/src/archive.rs index ee64631d..5ff1e83a 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -420,111 +420,3 @@ impl Archive { Ok(()) } } - -#[cfg(test)] -mod tests { - use std::fs; - use std::io::Read; - - use assert_fs::prelude::*; - use assert_fs::TempDir; - - use crate::test_fixtures::ScratchArchive; - - use super::*; - - #[test] - fn create_then_open_archive() { - let testdir = TempDir::new().unwrap(); - let arch_path = testdir.path().join("arch"); - let arch = Archive::create_path(&arch_path).unwrap(); - - assert!(arch.list_band_ids().unwrap().is_empty()); - - // We can re-open it. - Archive::open_path(&arch_path).unwrap(); - assert!(arch.list_band_ids().unwrap().is_empty()); - assert!(arch.last_complete_band().unwrap().is_none()); - } - - #[test] - fn fails_on_non_empty_directory() { - let temp = TempDir::new().unwrap(); - - temp.child("i am already here").touch().unwrap(); - - let result = Archive::create_path(temp.path()); - assert!(result.is_err()); - if let Err(Error::NewArchiveDirectoryNotEmpty) = result { - } else { - panic!("expected an error for a non-empty new archive directory") - } - - temp.close().unwrap(); - } - - /// A new archive contains just one header file. - /// The header is readable json containing only a version number. - #[test] - fn empty_archive() { - let af = ScratchArchive::new(); - - assert!(af.path().is_dir()); - assert!(af.path().join("CONSERVE").is_file()); - assert!(af.path().join("d").is_dir()); - - let header_path = af.path().join("CONSERVE"); - let mut header_file = fs::File::open(header_path).unwrap(); - let mut contents = String::new(); - header_file.read_to_string(&mut contents).unwrap(); - assert_eq!(contents, "{\"conserve_archive_version\":\"0.6\"}\n"); - - assert!( - af.last_band_id().unwrap().is_none(), - "Archive should have no bands yet" - ); - assert!( - af.last_complete_band().unwrap().is_none(), - "Archive should have no bands yet" - ); - assert_eq!( - af.referenced_blocks(&af.list_band_ids().unwrap()) - .unwrap() - .len(), - 0 - ); - assert_eq!(af.block_dir.block_names().unwrap().count(), 0); - } - - #[test] - fn create_bands() { - let af = ScratchArchive::new(); - assert!(af.path().join("d").is_dir()); - - // Make one band - let _band1 = Band::create(&af).unwrap(); - let band_path = af.path().join("b0000"); - assert!(band_path.is_dir()); - assert!(band_path.join("BANDHEAD").is_file()); - assert!(band_path.join("i").is_dir()); - - assert_eq!(af.list_band_ids().unwrap(), vec![BandId::new(&[0])]); - assert_eq!(af.last_band_id().unwrap(), Some(BandId::new(&[0]))); - - // Try creating a second band. - let _band2 = Band::create(&af).unwrap(); - assert_eq!( - af.list_band_ids().unwrap(), - vec![BandId::new(&[0]), BandId::new(&[1])] - ); - assert_eq!(af.last_band_id().unwrap(), Some(BandId::new(&[1]))); - - assert_eq!( - af.referenced_blocks(&af.list_band_ids().unwrap()) - .unwrap() - .len(), - 0 - ); - assert_eq!(af.block_dir.block_names().unwrap().count(), 0); - } -} diff --git a/tests/api/archive.rs b/tests/api/archive.rs new file mode 100644 index 00000000..d186fa9b --- /dev/null +++ b/tests/api/archive.rs @@ -0,0 +1,121 @@ +// Conserve backup system. +// Copyright 2015-2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! API tests for archives. + +use std::fs; +use std::io::Read; + +use assert_fs::prelude::*; +use assert_fs::TempDir; + +use conserve::archive::Archive; +use conserve::test_fixtures::ScratchArchive; +use conserve::Band; +use conserve::BandId; +use conserve::Error; + +#[test] +fn create_then_open_archive() { + let testdir = TempDir::new().unwrap(); + let arch_path = testdir.path().join("arch"); + let arch = Archive::create_path(&arch_path).unwrap(); + + assert!(arch.list_band_ids().unwrap().is_empty()); + + // We can re-open it. + Archive::open_path(&arch_path).unwrap(); + assert!(arch.list_band_ids().unwrap().is_empty()); + assert!(arch.last_complete_band().unwrap().is_none()); +} + +#[test] +fn fails_on_non_empty_directory() { + let temp = TempDir::new().unwrap(); + + temp.child("i am already here").touch().unwrap(); + + let result = Archive::create_path(temp.path()); + assert!(result.is_err()); + if let Err(Error::NewArchiveDirectoryNotEmpty) = result { + } else { + panic!("expected an error for a non-empty new archive directory") + } + + temp.close().unwrap(); +} + +/// A new archive contains just one header file. +/// The header is readable json containing only a version number. +#[test] +fn empty_archive() { + let af = ScratchArchive::new(); + + assert!(af.path().is_dir()); + assert!(af.path().join("CONSERVE").is_file()); + assert!(af.path().join("d").is_dir()); + + let header_path = af.path().join("CONSERVE"); + let mut header_file = fs::File::open(header_path).unwrap(); + let mut contents = String::new(); + header_file.read_to_string(&mut contents).unwrap(); + assert_eq!(contents, "{\"conserve_archive_version\":\"0.6\"}\n"); + + assert!( + af.last_band_id().unwrap().is_none(), + "Archive should have no bands yet" + ); + assert!( + af.last_complete_band().unwrap().is_none(), + "Archive should have no bands yet" + ); + assert_eq!( + af.referenced_blocks(&af.list_band_ids().unwrap()) + .unwrap() + .len(), + 0 + ); + assert_eq!(af.block_dir().block_names().unwrap().count(), 0); +} + +#[test] +fn create_bands() { + let af = ScratchArchive::new(); + assert!(af.path().join("d").is_dir()); + + // Make one band + let _band1 = Band::create(&af).unwrap(); + let band_path = af.path().join("b0000"); + assert!(band_path.is_dir()); + assert!(band_path.join("BANDHEAD").is_file()); + assert!(band_path.join("i").is_dir()); + + assert_eq!(af.list_band_ids().unwrap(), vec![BandId::new(&[0])]); + assert_eq!(af.last_band_id().unwrap(), Some(BandId::new(&[0]))); + + // Try creating a second band. + let _band2 = Band::create(&af).unwrap(); + assert_eq!( + af.list_band_ids().unwrap(), + vec![BandId::new(&[0]), BandId::new(&[1])] + ); + assert_eq!(af.last_band_id().unwrap(), Some(BandId::new(&[1]))); + + assert_eq!( + af.referenced_blocks(&af.list_band_ids().unwrap()) + .unwrap() + .len(), + 0 + ); + assert_eq!(af.block_dir().block_names().unwrap().count(), 0); +} diff --git a/tests/api/main.rs b/tests/api/main.rs index 1148bf2f..6f85f985 100644 --- a/tests/api/main.rs +++ b/tests/api/main.rs @@ -13,6 +13,7 @@ //! Tests for the Conserve library API. mod apath; +mod archive; mod backup; mod bandid; mod blockhash; From 017bddddab2a52c28e02fafb09926b46eabf67d6 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 16:11:40 -0700 Subject: [PATCH 36/86] Add Archive::iter_entries and use in tests --- src/archive.rs | 11 +++++++++++ src/show.rs | 2 +- src/stored_tree.rs | 1 + tests/damage/main.rs | 39 +++++++++++++-------------------------- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 5ff1e83a..a8a556fe 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -132,6 +132,17 @@ impl Archive { .map_err(Error::from) } + /// Return an iterator of entries in a selected version. + pub fn iter_entries( + &self, + band_selection: BandSelectionPolicy, + subtree: Apath, + exclude: Exclude, + ) -> Result> { + let stored_tree = self.open_stored_tree(band_selection)?; + stored_tree.iter_entries(subtree, exclude) + } + /// Returns a vector of band ids, in sorted order from first to last. pub fn list_band_ids(&self) -> Result> { let mut band_ids: Vec = self.iter_band_ids_unsorted()?.collect(); diff --git a/src/show.rs b/src/show.rs index 29ccc2ca..01e5a160 100644 --- a/src/show.rs +++ b/src/show.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2018, 2020, 2021, 2022 Martin Pool. +// Copyright 2018-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/src/stored_tree.rs b/src/stored_tree.rs index 62d416d0..1a074a7f 100644 --- a/src/stored_tree.rs +++ b/src/stored_tree.rs @@ -59,6 +59,7 @@ impl ReadTree for StoredTree { type IT = index::IndexEntryIter; /// Return an iter of index entries in this stored tree. + // TODO: Should return an iter of Result so that we can inspect them... fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result { Ok( IterStitchedIndexHunks::new(&self.archive, Some(self.band.id().clone())) diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 90c4e873..f7f98dc7 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -13,18 +13,15 @@ use assert_fs::prelude::*; use assert_fs::TempDir; -use conserve::Exclude; use dir_assert::assert_paths; -use indoc::indoc; use pretty_assertions::assert_eq; use rstest::rstest; use tracing_test::traced_test; // use predicates::prelude::*; -use conserve::show::show_entry_names; use conserve::{ - backup, restore, Apath, Archive, BackupOptions, BandId, ReadTree, RestoreOptions, - ValidateOptions, + backup, restore, Apath, Archive, BackupOptions, BandId, BandSelectionPolicy, Entry, Exclude, + RestoreOptions, ValidateOptions, }; mod strategy; @@ -73,27 +70,17 @@ fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damag assert_eq!(versions, [BandId::zero(), BandId::new(&[1])]); // Can list the contents of the second backup. - // let mut listing = String::new(); - // TODO: Better API for this! - let mut listing: Vec = Vec::new(); - show_entry_names( - archive - .open_stored_tree(conserve::BandSelectionPolicy::Latest) - .expect("open second backup") - .iter_entries(Apath::root(), Exclude::nothing()) - .expect("list entries"), - &mut listing, - false, - ) - .expect("show entry names"); - - assert_eq!( - String::from_utf8(listing).unwrap(), - indoc! {" - / - /file - "} - ); + let apaths = archive + .iter_entries( + BandSelectionPolicy::Latest, + Apath::root(), + Exclude::nothing(), + ) + .expect("iter entries") + .map(|entry| entry.apath().to_string()) + .collect::>(); + + assert_eq!(apaths, ["/", "/file"]); // Validation completes although with warnings. // TODO: This should return problems that we can inspect. From a0b250db5388438e5c8519aed695148524459572 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 16:40:54 -0700 Subject: [PATCH 37/86] Add a generic EntryValue Having separate types for live/stored trees might not be working so well. --- src/backup.rs | 15 +++--- src/change.rs | 4 +- src/entry.rs | 79 +++++++++++++++++++++++++++- src/index.rs | 6 +-- src/lib.rs | 4 +- src/live_tree.rs | 114 +++++++++++++---------------------------- tests/api/live_tree.rs | 7 +-- 7 files changed, 132 insertions(+), 97 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 713873aa..28bd9459 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -26,6 +26,7 @@ use tracing::error; use crate::blockdir::Address; use crate::change::Change; +use crate::entry::EntryValue; use crate::io::read_with_retries; use crate::progress::{Bar, Progress}; use crate::stats::{ @@ -195,7 +196,7 @@ impl BackupWriter { /// Return an indication of whether it changed (if it's a file), or /// None for non-plain-file types where that information is not currently /// calculated. - fn copy_entry(&mut self, entry: &LiveEntry, source: &LiveTree) -> Result> { + fn copy_entry(&mut self, entry: &EntryValue, source: &LiveTree) -> Result> { // TODO: Emit deletions for entries in the basis not present in the source. match entry.kind() { Kind::Dir => self.copy_dir(entry), @@ -221,7 +222,7 @@ impl BackupWriter { /// Copy in the contents of a file from another tree. fn copy_file( &mut self, - source_entry: &LiveEntry, + source_entry: &EntryValue, from_tree: &LiveTree, ) -> Result> { self.stats.files += 1; @@ -242,7 +243,7 @@ impl BackupWriter { result = Some(EntryChange::added(source_entry)); } let mut read_source = from_tree.file_contents(source_entry)?; - let size = source_entry.size().expect("LiveEntry has a size"); + let size = source_entry.size().expect("source entry has a size"); if size == 0 { self.index_builder .push_entry(IndexEntry::metadata_from(source_entry)); @@ -393,14 +394,14 @@ impl FileCombiner { /// Add the contents of a small file into this combiner. /// /// `entry` should be an IndexEntry that's complete apart from the block addresses. - fn push_file(&mut self, live_entry: &LiveEntry, from_file: &mut dyn Read) -> Result<()> { + fn push_file(&mut self, entry: &EntryValue, from_file: &mut dyn Read) -> Result<()> { let start = self.buf.len(); - let expected_len: usize = live_entry + let expected_len: usize = entry .size() .expect("small file has no length") .try_into() .unwrap(); - let index_entry = IndexEntry::metadata_from(live_entry); + let index_entry = IndexEntry::metadata_from(entry); if expected_len == 0 { self.stats.empty_files += 1; self.finished.push(index_entry); @@ -410,7 +411,7 @@ impl FileCombiner { let len = from_file .read(&mut self.buf[start..]) .map_err(|source| Error::StoreFile { - apath: live_entry.apath().to_owned(), + apath: entry.apath().to_owned(), source, })?; self.buf.truncate(start + len); diff --git a/src/change.rs b/src/change.rs index 9bc88b80..c4fbccb2 100644 --- a/src/change.rs +++ b/src/change.rs @@ -135,7 +135,7 @@ impl Change { /// Metadata about a changed entry other than its apath. #[derive(Debug, Clone, Eq, PartialEq, Serialize)] pub struct EntryMetadata { - // TODO: Eventually unify with LiveEntry or Entry? + // TODO: Eventually unify with EntryValue or Entry? #[serde(flatten)] pub kind: KindMetadata, pub mtime: OffsetDateTime, @@ -149,7 +149,7 @@ impl From<&dyn Entry> for EntryMetadata { EntryMetadata { kind: KindMetadata::from(entry), mtime: entry.mtime(), - owner: entry.owner(), + owner: entry.owner().clone(), unix_mode: entry.unix_mode(), } } diff --git a/src/entry.rs b/src/entry.rs index 04499621..b9fc0c13 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -23,6 +23,11 @@ use crate::owner::Owner; use crate::unix_mode::UnixMode; use crate::*; +/// A description of an file, directory, or symlink in a tree, independent +/// of whether it's recorded in a archive (an [IndexEntry]), or +/// in a source tree. +// TODO: Maybe keep this entirely in memory and explicitly look things +// up when needed. pub trait Entry: Debug { fn apath(&self) -> &Apath; fn kind(&self) -> Kind; @@ -30,5 +35,77 @@ pub trait Entry: Debug { fn size(&self) -> Option; fn symlink_target(&self) -> &Option; fn unix_mode(&self) -> UnixMode; - fn owner(&self) -> Owner; + fn owner(&self) -> &Owner; +} + +/// An in-memory [Entry] describing a file/dir/symlink, with no addresses. +#[derive(Debug)] +pub struct EntryValue { + pub(crate) apath: Apath, + pub(crate) kind: Kind, + pub(crate) mtime: OffsetDateTime, + pub(crate) size: Option, + pub(crate) symlink_target: Option, + pub(crate) unix_mode: UnixMode, + pub(crate) owner: Owner, +} + +impl Entry for EntryValue { + fn apath(&self) -> &Apath { + &self.apath + } + + fn kind(&self) -> Kind { + self.kind + } + + fn mtime(&self) -> OffsetDateTime { + self.mtime + } + + fn size(&self) -> Option { + self.size + } + + fn symlink_target(&self) -> &Option { + &self.symlink_target + } + + fn unix_mode(&self) -> UnixMode { + self.unix_mode + } + + fn owner(&self) -> &Owner { + &self.owner + } +} + +impl Entry for &EntryValue { + fn apath(&self) -> &Apath { + &self.apath + } + + fn kind(&self) -> Kind { + self.kind + } + + fn mtime(&self) -> OffsetDateTime { + self.mtime + } + + fn size(&self) -> Option { + self.size + } + + fn symlink_target(&self) -> &Option { + &self.symlink_target + } + + fn unix_mode(&self) -> UnixMode { + self.unix_mode + } + + fn owner(&self) -> &Owner { + &self.owner + } } diff --git a/src/index.rs b/src/index.rs index e8430daa..d283f52f 100644 --- a/src/index.rs +++ b/src/index.rs @@ -119,8 +119,8 @@ impl Entry for IndexEntry { self.unix_mode } - fn owner(&self) -> Owner { - self.owner.clone() + fn owner(&self) -> &Owner { + &self.owner } } @@ -140,7 +140,7 @@ impl IndexEntry { mtime: mtime.unix_timestamp(), mtime_nanos: mtime.nanosecond(), unix_mode: source.unix_mode(), - owner: source.owner(), + owner: source.owner().to_owned(), } } } diff --git a/src/lib.rs b/src/lib.rs index 8cf2ba7a..983a7db4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,7 @@ pub mod blockhash; pub mod change; pub mod compress; mod diff; -mod entry; +pub mod entry; pub mod errors; pub mod excludes; mod gc_lock; @@ -67,7 +67,7 @@ pub use crate::excludes::Exclude; pub use crate::gc_lock::GarbageCollectionLock; pub use crate::index::{IndexEntry, IndexRead, IndexWriter}; pub use crate::kind::Kind; -pub use crate::live_tree::{LiveEntry, LiveTree}; +pub use crate::live_tree::LiveTree; pub use crate::merge::MergeTrees; pub use crate::misc::bytes_to_human_mb; pub use crate::owner::Owner; diff --git a/src/live_tree.rs b/src/live_tree.rs index cfce2bcf..38247705 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -18,9 +18,9 @@ use std::fs; use std::io::ErrorKind; use std::path::{Path, PathBuf}; -use time::OffsetDateTime; use tracing::{error, warn}; +use crate::entry::EntryValue; use crate::owner::Owner; use crate::stats::LiveTreeIterStats; use crate::unix_mode::UnixMode; @@ -51,20 +51,8 @@ impl LiveTree { } } -/// An in-memory Entry describing a file/dir/symlink in a live tree. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct LiveEntry { - apath: Apath, - kind: Kind, - mtime: OffsetDateTime, - size: Option, - symlink_target: Option, - unix_mode: UnixMode, - owner: Owner, -} - impl tree::ReadTree for LiveTree { - type Entry = LiveEntry; + type Entry = EntryValue; type R = std::fs::File; type IT = Iter; @@ -72,7 +60,7 @@ impl tree::ReadTree for LiveTree { Iter::new(&self.path, subtree, exclude) } - fn file_contents(&self, entry: &LiveEntry) -> Result { + fn file_contents(&self, entry: &EntryValue) -> Result { assert_eq!(entry.kind(), Kind::File); let path = self.relative_path(&entry.apath); fs::File::open(&path).map_err(|source| Error::ReadSourceFile { path, source }) @@ -88,63 +76,31 @@ impl tree::ReadTree for LiveTree { } } -impl Entry for LiveEntry { - fn apath(&self) -> &Apath { - &self.apath - } - - fn kind(&self) -> Kind { - self.kind - } - - fn mtime(&self) -> OffsetDateTime { - self.mtime - } - - fn size(&self) -> Option { - self.size - } - - fn symlink_target(&self) -> &Option { - &self.symlink_target - } - - fn unix_mode(&self) -> UnixMode { - self.unix_mode - } - - fn owner(&self) -> Owner { - self.owner.clone() - } -} - -impl LiveEntry { - fn from_fs_metadata( - apath: Apath, - metadata: &fs::Metadata, - symlink_target: Option, - ) -> LiveEntry { - // TODO: Could we read the symlink target here, rather than in the caller? - let mtime = metadata - .modified() - .expect("Failed to get file mtime") - .into(); - let size = if metadata.is_file() { - Some(metadata.len()) - } else { - None - }; - let owner = Owner::from(metadata); - let unix_mode = UnixMode::from(metadata.permissions()); - LiveEntry { - apath, - kind: metadata.file_type().into(), - mtime, - symlink_target, - size, - unix_mode, - owner, - } +fn entry_from_fs_metadata( + apath: Apath, + metadata: &fs::Metadata, + symlink_target: Option, +) -> EntryValue { + // TODO: Could we read the symlink target here, rather than in the caller? + let mtime = metadata + .modified() + .expect("Failed to get file mtime") + .into(); + let size = if metadata.is_file() { + Some(metadata.len()) + } else { + None + }; + let owner = Owner::from(metadata); + let unix_mode = UnixMode::from(metadata.permissions()); + EntryValue { + apath, + kind: metadata.file_type().into(), + mtime, + symlink_target, + size, + unix_mode, + owner, } } @@ -166,7 +122,7 @@ pub struct Iter { /// All entries that have been seen but not yet returned by the iterator, in the order they /// should be returned. - entry_deque: VecDeque, + entry_deque: VecDeque, /// Check that emitted paths are in the right order. check_order: apath::DebugCheckOrder, @@ -183,7 +139,7 @@ impl Iter { fn new(root_path: &Path, subtree: Apath, exclude: Exclude) -> Result { let start_metadata = fs::symlink_metadata(subtree.below(root_path))?; // Preload iter to return the root and then recurse into it. - let entry_deque: VecDeque = [LiveEntry::from_fs_metadata( + let entry_deque: VecDeque = [entry_from_fs_metadata( subtree.clone(), &start_metadata, None, @@ -209,7 +165,7 @@ impl Iter { fn visit_next_directory(&mut self, parent_apath: &Apath) { self.stats.directories_visited += 1; // Tuples of (name, entry) so that we can sort children by name. - let mut children = Vec::<(String, LiveEntry)>::new(); + let mut children = Vec::<(String, EntryValue)>::new(); let dir_path = parent_apath.below(&self.root_path); let dir_iter = match fs::read_dir(&dir_path) { Ok(i) => i, @@ -279,7 +235,7 @@ impl Iter { } }; - // TODO: Move this into LiveEntry::from_fs_metadata, once there's a + // TODO: Move this into entry_from_fs_metadata, once there's a // global way for it to complain about errors. let target: Option = if ft.is_symlink() { let t = match dir_path.join(dir_entry.file_name()).read_link() { @@ -304,7 +260,7 @@ impl Iter { } children.push(( child_name.to_string(), - LiveEntry::from_fs_metadata(child_apath, &metadata, target), + entry_from_fs_metadata(child_apath, &metadata, target), )); } // To get the right overall tree ordering, any new subdirectories @@ -332,9 +288,9 @@ impl Iter { // subdirectories are then visited, also in sorted order, before returning to // any higher-level directories. impl Iterator for Iter { - type Item = LiveEntry; + type Item = EntryValue; - fn next(&mut self) -> Option { + fn next(&mut self) -> Option { loop { if let Some(entry) = self.entry_deque.pop_front() { // Have already found some entries, so just return the first. diff --git a/tests/api/live_tree.rs b/tests/api/live_tree.rs index 645144be..917f6592 100644 --- a/tests/api/live_tree.rs +++ b/tests/api/live_tree.rs @@ -12,6 +12,7 @@ use pretty_assertions::assert_eq; +use conserve::entry::EntryValue; use conserve::test_fixtures::TreeFixture; use conserve::*; @@ -32,11 +33,11 @@ fn list_simple_directory() { tf.create_dir("jelly"); tf.create_dir("jam/.etc"); let lt = LiveTree::open(tf.path()).unwrap(); - let result: Vec = lt + let result: Vec = lt .iter_entries(Apath::root(), Exclude::nothing()) .unwrap() .collect(); - let names = entry_iter_to_apath_strings(result.clone()); + let names = entry_iter_to_apath_strings(&result); // First one is the root assert_eq!( names, @@ -53,7 +54,7 @@ fn list_simple_directory() { let repr = format!("{:?}", &result[6]); println!("{repr}"); - assert!(repr.starts_with("LiveEntry {")); + assert!(repr.starts_with("EntryValue {")); assert!(repr.contains("Apath(\"/jam/apricot\")")); // TODO: Somehow get the stats out of the iterator. From fafeef9f33ca15e8dc5aaac05daecc50d8e2b4c2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 17:06:37 -0700 Subject: [PATCH 38/86] Use EntryValue more places --- src/backup.rs | 6 +++--- src/bin/conserve.rs | 32 ++++++++++++++++---------------- src/change.rs | 20 ++++++++++---------- src/entry.rs | 6 +++--- src/index.rs | 26 ++++++++++++++++++++++++-- src/lib.rs | 2 +- src/merge.rs | 20 ++++++++++---------- src/restore.rs | 5 ++--- src/show.rs | 2 +- src/tree.rs | 2 +- tests/api/live_tree.rs | 2 +- tests/damage/main.rs | 4 ++-- 12 files changed, 74 insertions(+), 53 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 28bd9459..52be8c50 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -212,7 +212,7 @@ impl BackupWriter { } } - fn copy_dir(&mut self, source_entry: &E) -> Result> { + fn copy_dir(&mut self, source_entry: &EntryValue) -> Result> { self.stats.directories += 1; self.index_builder .push_entry(IndexEntry::metadata_from(source_entry)); @@ -268,7 +268,7 @@ impl BackupWriter { Ok(result) } - fn copy_symlink(&mut self, source_entry: &E) -> Result> { + fn copy_symlink(&mut self, source_entry: &EntryValue) -> Result> { let target = source_entry.symlink_target().clone(); self.stats.symlinks += 1; assert!(target.is_some()); @@ -441,7 +441,7 @@ impl FileCombiner { /// not changed, without reading the file content. /// /// Caution: this does not check the symlink target. -fn entry_metadata_unchanged(new_entry: &E, basis_entry: &O) -> bool { +fn entry_metadata_unchanged(new_entry: &E, basis_entry: &O) -> bool { basis_entry.kind() == new_entry.kind() && basis_entry.mtime() == new_entry.mtime() && basis_entry.size() == new_entry.size() diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index 3c8a2499..ea97770d 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -414,22 +414,22 @@ impl Command { long_listing, } => { let exclude = Exclude::from_patterns_and_files(exclude, exclude_from)?; - if let Some(archive) = &stos.archive { - // TODO: Option for subtree. - show::show_entry_names( - stored_tree_from_opt(archive, &stos.backup)? - .iter_entries(Apath::root(), exclude)?, - &mut stdout, - *long_listing, - )?; - } else { - show::show_entry_names( - LiveTree::open(stos.source.clone().unwrap())? - .iter_entries(Apath::root(), exclude)?, - &mut stdout, - *long_listing, - )?; - } + let entry_iter: Box> = + if let Some(archive) = &stos.archive { + // TODO: Option for subtree. + Box::new( + stored_tree_from_opt(archive, &stos.backup)? + .iter_entries(Apath::root(), exclude)? + .map(|it| it.into()), + ) + } else { + Box::new( + LiveTree::open(stos.source.clone().unwrap())? + .iter_entries(Apath::root(), exclude)?, + ) + }; + + show::show_entry_names(entry_iter, &mut stdout, *long_listing)?; } Command::Restore { archive, diff --git a/src/change.rs b/src/change.rs index c4fbccb2..09bd15fb 100644 --- a/src/change.rs +++ b/src/change.rs @@ -18,7 +18,7 @@ use std::fmt; use serde::Serialize; use time::OffsetDateTime; -use crate::{Apath, Entry, Kind, Owner, Result, UnixMode}; +use crate::{Apath, EntryTrait, Kind, Owner, Result, UnixMode}; /// Summary of some kind of change to an entry from backup, diff, restore, etc. #[derive(Debug, Clone, Eq, PartialEq, Serialize)] @@ -33,7 +33,7 @@ impl EntryChange { self.change.is_unchanged() } - pub(crate) fn diff_metadata(a: &dyn Entry, b: &dyn Entry) -> Self { + pub(crate) fn diff_metadata(a: &AE, b: &BE) -> Self { debug_assert_eq!(a.apath(), b.apath()); let ak = a.kind(); // mtime is only treated as a significant change for files, because @@ -51,7 +51,7 @@ impl EntryChange { } } - pub(crate) fn added(entry: &dyn Entry) -> Self { + pub(crate) fn added(entry: &dyn EntryTrait) -> Self { EntryChange { apath: entry.apath().clone(), change: Change::Added { @@ -61,7 +61,7 @@ impl EntryChange { } #[allow(unused)] // Never generated in backups at the moment - pub(crate) fn deleted(entry: &dyn Entry) -> Self { + pub(crate) fn deleted(entry: &dyn EntryTrait) -> Self { EntryChange { apath: entry.apath().clone(), change: Change::Deleted { @@ -70,7 +70,7 @@ impl EntryChange { } } - pub(crate) fn unchanged(entry: &dyn Entry) -> Self { + pub(crate) fn unchanged(entry: &dyn EntryTrait) -> Self { EntryChange { apath: entry.apath().clone(), change: Change::Unchanged { @@ -79,7 +79,7 @@ impl EntryChange { } } - pub(crate) fn changed(old: &dyn Entry, new: &dyn Entry) -> Self { + pub(crate) fn changed(old: &dyn EntryTrait, new: &dyn EntryTrait) -> Self { debug_assert_eq!(old.apath(), new.apath()); EntryChange { apath: old.apath().clone(), @@ -144,8 +144,8 @@ pub struct EntryMetadata { pub unix_mode: UnixMode, } -impl From<&dyn Entry> for EntryMetadata { - fn from(entry: &dyn Entry) -> Self { +impl From<&dyn EntryTrait> for EntryMetadata { + fn from(entry: &dyn EntryTrait) -> Self { EntryMetadata { kind: KindMetadata::from(entry), mtime: entry.mtime(), @@ -163,8 +163,8 @@ pub enum KindMetadata { Symlink { target: String }, } -impl From<&dyn Entry> for KindMetadata { - fn from(entry: &dyn Entry) -> Self { +impl From<&dyn EntryTrait> for KindMetadata { + fn from(entry: &dyn EntryTrait) -> Self { match entry.kind() { Kind::File => KindMetadata::File { size: entry.size().unwrap(), diff --git a/src/entry.rs b/src/entry.rs index b9fc0c13..f4ad2aa2 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -28,7 +28,7 @@ use crate::*; /// in a source tree. // TODO: Maybe keep this entirely in memory and explicitly look things // up when needed. -pub trait Entry: Debug { +pub trait EntryTrait: Debug { fn apath(&self) -> &Apath; fn kind(&self) -> Kind; fn mtime(&self) -> OffsetDateTime; @@ -50,7 +50,7 @@ pub struct EntryValue { pub(crate) owner: Owner, } -impl Entry for EntryValue { +impl EntryTrait for EntryValue { fn apath(&self) -> &Apath { &self.apath } @@ -80,7 +80,7 @@ impl Entry for EntryValue { } } -impl Entry for &EntryValue { +impl EntryTrait for &EntryValue { fn apath(&self) -> &Apath { &self.apath } diff --git a/src/index.rs b/src/index.rs index d283f52f..5ae644c9 100644 --- a/src/index.rs +++ b/src/index.rs @@ -25,6 +25,7 @@ use time::OffsetDateTime; use tracing::error; use crate::compress::snappy::{Compressor, Decompressor}; +use crate::entry::EntryValue; use crate::kind::Kind; use crate::owner::Owner; use crate::stats::{IndexReadStats, IndexWriterStats}; @@ -88,7 +89,28 @@ pub struct IndexEntry { } // GRCOV_EXCLUDE_STOP -impl Entry for IndexEntry { +impl From for EntryValue { + fn from(index_entry: IndexEntry) -> EntryValue { + EntryValue { + apath: index_entry.apath, + kind: index_entry.kind, + mtime: OffsetDateTime::from_unix_seconds_and_nanos( + index_entry.mtime, + index_entry.mtime_nanos, + ), + size: if index_entry.kind == Kind::File { + Some(index_entry.addrs.iter().map(|a| a.len).sum()) + } else { + None + }, + symlink_target: index_entry.target, + unix_mode: index_entry.unix_mode, + owner: index_entry.owner, + } + } +} + +impl EntryTrait for IndexEntry { /// Return apath relative to the top of the tree. fn apath(&self) -> &Apath { &self.apath @@ -126,7 +148,7 @@ impl Entry for IndexEntry { impl IndexEntry { /// Copy the metadata, but not the body content, from another entry. - pub(crate) fn metadata_from(source: &E) -> IndexEntry { + pub(crate) fn metadata_from(source: &EntryValue) -> IndexEntry { let mtime = source.mtime(); assert_eq!( source.symlink_target().is_some(), diff --git a/src/lib.rs b/src/lib.rs index 983a7db4..3e51c466 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,7 +61,7 @@ pub use crate::blockdir::BlockDir; pub use crate::blockhash::BlockHash; pub use crate::change::{ChangeCallback, EntryChange}; pub use crate::diff::{diff, DiffOptions}; -pub use crate::entry::Entry; +pub use crate::entry::{EntryTrait, EntryValue}; pub use crate::errors::Error; pub use crate::excludes::Exclude; pub use crate::gc_lock::GarbageCollectionLock; diff --git a/src/merge.rs b/src/merge.rs index 10686820..2cbbfa9e 100644 --- a/src/merge.rs +++ b/src/merge.rs @@ -28,8 +28,8 @@ use crate::*; #[derive(Debug, PartialEq, Eq)] pub enum MatchedEntries where - AE: Entry, - BE: Entry, + AE: EntryTrait, + BE: EntryTrait, { Left(AE), Right(BE), @@ -38,8 +38,8 @@ where impl MatchedEntries where - AE: Entry, - BE: Entry, + AE: EntryTrait, + BE: EntryTrait, { pub(crate) fn to_entry_change(&self) -> EntryChange { match self { @@ -56,8 +56,8 @@ where /// side, not whether there is a content difference. pub struct MergeTrees where - AE: Entry, - BE: Entry, + AE: EntryTrait, + BE: EntryTrait, AIT: Iterator, BIT: Iterator, { @@ -71,8 +71,8 @@ where impl MergeTrees where - AE: Entry, - BE: Entry, + AE: EntryTrait, + BE: EntryTrait, AIT: Iterator, BIT: Iterator, { @@ -88,8 +88,8 @@ where impl Iterator for MergeTrees where - AE: Entry, - BE: Entry, + AE: EntryTrait, + BE: EntryTrait, AIT: Iterator, BIT: Iterator, { diff --git a/src/restore.rs b/src/restore.rs index 1f4a75a1..ac08768c 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -27,7 +27,6 @@ use time::OffsetDateTime; use tracing::{error, warn}; use crate::band::BandSelectionPolicy; -use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; use crate::progress::{Bar, Progress}; use crate::stats::RestoreStats; @@ -240,7 +239,7 @@ fn copy_file( } #[cfg(unix)] -fn restore_symlink(path: &Path, entry: &E) -> Result<()> { +fn restore_symlink(path: &Path, entry: &IndexEntry) -> Result<()> { use std::os::unix::fs as unix_fs; if let Some(ref target) = entry.symlink_target() { if let Err(source) = unix_fs::symlink(target, path) { @@ -263,7 +262,7 @@ fn restore_symlink(path: &Path, entry: &E) -> Result<()> { } #[cfg(not(unix))] -fn restore_symlink(_restore_path: &Path, entry: &E) -> Result<()> { +fn restore_symlink(_restore_path: &Path, entry: &IndexEntry) -> Result<()> { // TODO: Add a test with a canned index containing a symlink, and expect // it cannot be restored on Windows and can be on Unix. warn!("Can't restore symlinks on non-Unix: {}", entry.apath()); diff --git a/src/show.rs b/src/show.rs index 01e5a160..cd847a21 100644 --- a/src/show.rs +++ b/src/show.rs @@ -128,7 +128,7 @@ pub fn show_index_json(band: &Band, w: &mut dyn Write) -> Result<()> { .map_err(|source| Error::SerializeIndex { source }) } -pub fn show_entry_names>( +pub fn show_entry_names>( it: I, w: &mut dyn Write, long_listing: bool, diff --git a/src/tree.rs b/src/tree.rs index aeb8a2c0..ad51f270 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -21,7 +21,7 @@ use crate::*; /// Abstract Tree that may be either on the real filesystem or stored in an archive. pub trait ReadTree { - type Entry: Entry + 'static; + type Entry: EntryTrait + 'static; type R: std::io::Read; type IT: Iterator; diff --git a/tests/api/live_tree.rs b/tests/api/live_tree.rs index 917f6592..4444fe75 100644 --- a/tests/api/live_tree.rs +++ b/tests/api/live_tree.rs @@ -138,7 +138,7 @@ fn exclude_cachedir() { fn entry_iter_to_apath_strings(entry_iter: EntryIter) -> Vec where EntryIter: IntoIterator, - E: Entry, + E: EntryTrait, { entry_iter .into_iter() diff --git a/tests/damage/main.rs b/tests/damage/main.rs index f7f98dc7..3494da05 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -20,8 +20,8 @@ use tracing_test::traced_test; // use predicates::prelude::*; use conserve::{ - backup, restore, Apath, Archive, BackupOptions, BandId, BandSelectionPolicy, Entry, Exclude, - RestoreOptions, ValidateOptions, + backup, restore, Apath, Archive, BackupOptions, BandId, BandSelectionPolicy, EntryTrait, + Exclude, RestoreOptions, ValidateOptions, }; mod strategy; From fa91d484ce11cfe89f36b8446357ee054c65d73b Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 17:15:26 -0700 Subject: [PATCH 39/86] Allow serializing the EntryValue --- src/entry.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/entry.rs b/src/entry.rs index f4ad2aa2..cfcc38ab 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -16,6 +16,7 @@ use std::fmt::Debug; +use serde::Serialize; use time::OffsetDateTime; use crate::kind::Kind; @@ -39,14 +40,17 @@ pub trait EntryTrait: Debug { } /// An in-memory [Entry] describing a file/dir/symlink, with no addresses. -#[derive(Debug)] +#[derive(Debug, Serialize, Clone, Eq, PartialEq)] pub struct EntryValue { pub(crate) apath: Apath, + // TODO: Maybe a KindMetadata, so that we only have a size for files + // and a target for symlinks? pub(crate) kind: Kind, pub(crate) mtime: OffsetDateTime, pub(crate) size: Option, pub(crate) symlink_target: Option, pub(crate) unix_mode: UnixMode, + #[serde(flatten)] pub(crate) owner: Owner, } From a1c074b02cbfbb74c5aa7effc43f881fbfdc39e7 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 17:35:32 -0700 Subject: [PATCH 40/86] Remove Tree::file_contents Exposing it as io::Read seems inefficient: most users will know what they want, I think, and it simplifies this interface that isn't sitting naturally in Rust... --- src/backup.rs | 6 +++--- src/live_tree.rs | 15 ++++++++------- src/restore.rs | 15 ++++++++------- src/stored_file.rs | 5 ++--- src/stored_tree.rs | 10 +++------- src/tree.rs | 7 +------ tests/api/backup.rs | 4 ++-- 7 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 52be8c50..cd41e8c0 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -242,7 +242,6 @@ impl BackupWriter { self.stats.new_files += 1; result = Some(EntryChange::added(source_entry)); } - let mut read_source = from_tree.file_contents(source_entry)?; let size = source_entry.size().expect("source entry has a size"); if size == 0 { self.index_builder @@ -250,14 +249,15 @@ impl BackupWriter { self.stats.empty_files += 1; return Ok(result); } + let mut source_file = from_tree.open_file(source_entry)?; if size <= SMALL_FILE_CAP { self.file_combiner - .push_file(source_entry, &mut read_source)?; + .push_file(source_entry, &mut source_file)?; return Ok(result); } let addrs = store_file_content( apath, - &mut read_source, + &mut source_file, &mut self.block_dir, &mut self.stats, )?; diff --git a/src/live_tree.rs b/src/live_tree.rs index 38247705..3fe61c41 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -15,6 +15,7 @@ use std::collections::vec_deque::VecDeque; use std::fs; +use std::fs::File; use std::io::ErrorKind; use std::path::{Path, PathBuf}; @@ -49,23 +50,23 @@ impl LiveTree { pub fn path(&self) -> &Path { &self.path } + + /// Open a file inside the tree to read. + pub fn open_file(&self, entry: &EntryValue) -> Result { + assert_eq!(entry.kind(), Kind::File); + let path = self.relative_path(&entry.apath); + fs::File::open(&path).map_err(|source| Error::ReadSourceFile { path, source }) + } } impl tree::ReadTree for LiveTree { type Entry = EntryValue; - type R = std::fs::File; type IT = Iter; fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result { Iter::new(&self.path, subtree, exclude) } - fn file_contents(&self, entry: &EntryValue) -> Result { - assert_eq!(entry.kind(), Kind::File); - let path = self.relative_path(&entry.apath); - fs::File::open(&path).map_err(|source| Error::ReadSourceFile { path, source }) - } - fn estimate_count(&self) -> Result { // TODO: This stats the file and builds an entry about them, just to // throw it away. We could perhaps change the iter to optionally do diff --git a/src/restore.rs b/src/restore.rs index ac08768c..dd480eb5 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -124,7 +124,7 @@ pub fn restore( Kind::File => { stats.files += 1; increment_counter!("conserve.restore.files"); - match copy_file(path.clone(), &entry, &st) { + match restore_file(path.clone(), &entry, &st) { Err(err) => { error!(?err, ?path, "Failed to restore file"); stats.errors += 1; @@ -195,10 +195,10 @@ fn apply_deferrals(deferrals: &[DirDeferral]) -> Result { } /// Copy in the contents of a file from another tree. -fn copy_file( +fn restore_file( path: PathBuf, - source_entry: &R::Entry, - from_tree: &R, + source_entry: &IndexEntry, + from_tree: &StoredTree, ) -> Result { let restore_err = |source| Error::Restore { path: path.clone(), @@ -206,9 +206,10 @@ fn copy_file( }; let mut stats = RestoreStats::default(); let mut restore_file = File::create(&path).map_err(restore_err)?; - // TODO: Read one block at a time: don't pull all the contents into memory. - let content = &mut from_tree.file_contents(source_entry)?; - let len = std::io::copy(content, &mut restore_file).map_err(restore_err)?; + // TODO: Read one block at a time, maybe don't go through io::copy. + let stored_file = from_tree.open_stored_file(source_entry); + let len = + std::io::copy(&mut stored_file.into_read(), &mut restore_file).map_err(restore_err)?; stats.uncompressed_file_bytes = len; counter!("conserve.restore.file_bytes", len); restore_file.flush().map_err(restore_err)?; diff --git a/src/stored_file.rs b/src/stored_file.rs index 2127724b..506e33c4 100644 --- a/src/stored_file.rs +++ b/src/stored_file.rs @@ -16,8 +16,7 @@ use crate::*; /// Returns the contents of a file stored in the archive, as an iter of byte blocks. /// -/// These can be constructed through `StoredTree::open_stored_file()` or more -/// generically through `ReadTree::file_contents`. +/// These can be constructed through `StoredTree::open_stored_file()`. pub struct StoredFile { block_dir: BlockDir, @@ -32,7 +31,7 @@ impl StoredFile { } /// Open a cursor on this file that implements `std::io::Read`. - pub(crate) fn into_read(self) -> ReadStoredFile { + pub fn into_read(self) -> ReadStoredFile { ReadStoredFile { remaining_addrs: self.addrs.into_iter(), buf: Vec::::new(), diff --git a/src/stored_tree.rs b/src/stored_tree.rs index 1a074a7f..e5cf28fa 100644 --- a/src/stored_tree.rs +++ b/src/stored_tree.rs @@ -20,7 +20,7 @@ use crate::blockdir::BlockDir; use crate::stitch::IterStitchedIndexHunks; -use crate::stored_file::{ReadStoredFile, StoredFile}; +use crate::stored_file::StoredFile; use crate::*; /// Read index and file contents for a version stored in the archive. @@ -48,13 +48,13 @@ impl StoredTree { } /// Open a file stored within this tree. - fn open_stored_file(&self, entry: &IndexEntry) -> StoredFile { + pub fn open_stored_file(&self, entry: &IndexEntry) -> StoredFile { + assert_eq!(entry.kind(), Kind::File); StoredFile::open(self.block_dir.clone(), entry.addrs.clone()) } } impl ReadTree for StoredTree { - type R = ReadStoredFile; type Entry = IndexEntry; type IT = index::IndexEntryIter; @@ -67,10 +67,6 @@ impl ReadTree for StoredTree { ) } - fn file_contents(&self, entry: &Self::Entry) -> Result { - Ok(self.open_stored_file(entry).into_read()) - } - fn estimate_count(&self) -> Result { self.band.index().estimate_entry_count() } diff --git a/src/tree.rs b/src/tree.rs index ad51f270..b35e3422 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2017, 2018, 2019, 2020, 2022 Martin Pool. +// Copyright 2017-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -22,7 +22,6 @@ use crate::*; /// Abstract Tree that may be either on the real filesystem or stored in an archive. pub trait ReadTree { type Entry: EntryTrait + 'static; - type R: std::io::Read; type IT: Iterator; /// Iterate, in apath order, all the entries in this tree. @@ -32,10 +31,6 @@ pub trait ReadTree { /// iterator. fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result; - /// Read file contents as a `std::io::Read`. - // TODO: Remove this and use ReadBlocks or similar. - fn file_contents(&self, entry: &Self::Entry) -> Result; - /// Estimate the number of entries in the tree. /// This might do somewhat expensive IO, so isn't the Iter's `size_hint`. fn estimate_count(&self) -> Result; diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 4dbfd925..a8f01249 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -300,9 +300,9 @@ pub fn empty_file_uses_zero_blocks() { .unwrap() .find(|i| &i.apath == "/empty") .expect("found one entry"); - let mut sf = st.file_contents(&empty_entry).unwrap(); + let stored_file = st.open_stored_file(&empty_entry); let mut s = String::new(); - assert_eq!(sf.read_to_string(&mut s).unwrap(), 0); + assert_eq!(stored_file.into_read().read_to_string(&mut s).unwrap(), 0); assert_eq!(s.len(), 0); // Restore it From ce0a058422352ef5f5f0e748a85c5ae61d7c41ad Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 17:45:34 -0700 Subject: [PATCH 41/86] Factor out entry_iter_to_apath_strings --- src/test_fixtures.rs | 14 ++++++++++++++ tests/api/live_tree.rs | 16 +--------------- tests/damage/main.rs | 6 +++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test_fixtures.rs b/src/test_fixtures.rs index e8a2c18e..af2dec15 100644 --- a/src/test_fixtures.rs +++ b/src/test_fixtures.rs @@ -186,3 +186,17 @@ impl Default for TreeFixture { Self::new() } } + +/// Collect apaths from an iterator into a list of string. +/// +/// This is more loosely typed but useful for tests. +pub fn entry_iter_to_apath_strings(entry_iter: EntryIter) -> Vec +where + EntryIter: IntoIterator, + E: EntryTrait, +{ + entry_iter + .into_iter() + .map(|entry| entry.apath().clone().into()) + .collect() +} diff --git a/tests/api/live_tree.rs b/tests/api/live_tree.rs index 4444fe75..afb62846 100644 --- a/tests/api/live_tree.rs +++ b/tests/api/live_tree.rs @@ -13,7 +13,7 @@ use pretty_assertions::assert_eq; use conserve::entry::EntryValue; -use conserve::test_fixtures::TreeFixture; +use conserve::test_fixtures::{entry_iter_to_apath_strings, TreeFixture}; use conserve::*; #[test] @@ -131,17 +131,3 @@ fn exclude_cachedir() { entry_iter_to_apath_strings(lt.iter_entries(Apath::root(), Exclude::nothing()).unwrap()); assert_eq!(names, ["/", "/a"]); } - -/// Collect apaths from an iterator into a list of string. -/// -/// This is more loosely typed but useful for tests. -fn entry_iter_to_apath_strings(entry_iter: EntryIter) -> Vec -where - EntryIter: IntoIterator, - E: EntryTrait, -{ - entry_iter - .into_iter() - .map(|entry| entry.apath().clone().into()) - .collect() -} diff --git a/tests/damage/main.rs b/tests/damage/main.rs index 3494da05..c38e6224 100644 --- a/tests/damage/main.rs +++ b/tests/damage/main.rs @@ -70,15 +70,15 @@ fn backup_after_damage(#[values(Damage::Delete, Damage::Truncate)] damage: Damag assert_eq!(versions, [BandId::zero(), BandId::new(&[1])]); // Can list the contents of the second backup. - let apaths = archive + let apaths: Vec = archive .iter_entries( BandSelectionPolicy::Latest, Apath::root(), Exclude::nothing(), ) .expect("iter entries") - .map(|entry| entry.apath().to_string()) - .collect::>(); + .map(|e| e.apath().to_string()) + .collect(); assert_eq!(apaths, ["/", "/file"]); From 4501e45f5b03ab5d131733b76b904c9e1311e443 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 18:24:50 -0700 Subject: [PATCH 42/86] More refactor of Entry handling --- src/change.rs | 2 +- src/entry.rs | 90 ++++++++++++++++++++++++------------------------ src/errors.rs | 6 ++++ src/index.rs | 29 ++++++++++------ src/live_tree.rs | 85 ++++++++++++++++++++++++--------------------- 5 files changed, 116 insertions(+), 96 deletions(-) diff --git a/src/change.rs b/src/change.rs index 09bd15fb..5f792a1f 100644 --- a/src/change.rs +++ b/src/change.rs @@ -171,7 +171,7 @@ impl From<&dyn EntryTrait> for KindMetadata { }, Kind::Dir => KindMetadata::Dir, Kind::Symlink => KindMetadata::Symlink { - target: entry.symlink_target().clone().unwrap(), + target: entry.symlink_target().unwrap().to_owned(), }, Kind::Unknown => panic!("unexpected Kind::Unknown on {:?}", entry.apath()), } diff --git a/src/entry.rs b/src/entry.rs index cfcc38ab..1fea0be6 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -14,6 +14,7 @@ //! An entry representing a file, directory, etc, in either a //! stored tree or local tree. +use std::borrow::Borrow; use std::fmt::Debug; use serde::Serialize; @@ -34,82 +35,81 @@ pub trait EntryTrait: Debug { fn kind(&self) -> Kind; fn mtime(&self) -> OffsetDateTime; fn size(&self) -> Option; - fn symlink_target(&self) -> &Option; + fn symlink_target(&self) -> Option<&str>; fn unix_mode(&self) -> UnixMode; fn owner(&self) -> &Owner; } +/// Per-kind metadata. +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +#[serde(tag = "kind")] +pub enum KindMeta { + File { size: u64 }, + Dir, + Symlink { target: String }, + Unknown, +} + +impl From<&KindMeta> for Kind { + fn from(from: &KindMeta) -> Kind { + match from { + KindMeta::Dir => Kind::Dir, + KindMeta::File { .. } => Kind::File, + KindMeta::Symlink { .. } => Kind::Symlink, + KindMeta::Unknown => Kind::Unknown, + } + } +} + /// An in-memory [Entry] describing a file/dir/symlink, with no addresses. #[derive(Debug, Serialize, Clone, Eq, PartialEq)] pub struct EntryValue { pub(crate) apath: Apath, - // TODO: Maybe a KindMetadata, so that we only have a size for files - // and a target for symlinks? - pub(crate) kind: Kind, + + /// Is it a file, dir, or symlink, and for files the size and for symlinks the target. + #[serde(flatten)] + pub(crate) kind_meta: KindMeta, + + /// Modification time. pub(crate) mtime: OffsetDateTime, - pub(crate) size: Option, - pub(crate) symlink_target: Option, pub(crate) unix_mode: UnixMode, #[serde(flatten)] pub(crate) owner: Owner, } -impl EntryTrait for EntryValue { - fn apath(&self) -> &Apath { - &self.apath - } - - fn kind(&self) -> Kind { - self.kind - } - - fn mtime(&self) -> OffsetDateTime { - self.mtime - } - - fn size(&self) -> Option { - self.size - } - - fn symlink_target(&self) -> &Option { - &self.symlink_target - } - - fn unix_mode(&self) -> UnixMode { - self.unix_mode - } - - fn owner(&self) -> &Owner { - &self.owner - } -} - -impl EntryTrait for &EntryValue { +impl + Debug> EntryTrait for B { fn apath(&self) -> &Apath { - &self.apath + &self.borrow().apath } fn kind(&self) -> Kind { - self.kind + Kind::from(&self.borrow().kind_meta) } fn mtime(&self) -> OffsetDateTime { - self.mtime + self.borrow().mtime } fn size(&self) -> Option { - self.size + if let KindMeta::File { size } = self.borrow().kind_meta { + Some(size) + } else { + None + } } - fn symlink_target(&self) -> &Option { - &self.symlink_target + fn symlink_target(&self) -> Option<&str> { + match &self.borrow().kind_meta { + KindMeta::Symlink { target } => Some(target), + _ => None, + } } fn unix_mode(&self) -> UnixMode { - self.unix_mode + self.borrow().unix_mode } fn owner(&self) -> &Owner { - &self.owner + &self.borrow().owner } } diff --git a/src/errors.rs b/src/errors.rs index 3cb12f80..f8df0977 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -244,6 +244,12 @@ pub enum Error { source: io::Error, }, + #[error("Unsupported source file kind: {path:?}")] + UnsupportedSourceKind { path: PathBuf }, + + #[error("Unsupported symlink encoding: {path:?}")] + UnsupportedTargetEncoding { path: PathBuf }, + #[error("Failed to read source tree {:?}", path)] ListSourceTree { path: PathBuf, diff --git a/src/index.rs b/src/index.rs index 5ae644c9..e7f55a7b 100644 --- a/src/index.rs +++ b/src/index.rs @@ -25,7 +25,7 @@ use time::OffsetDateTime; use tracing::error; use crate::compress::snappy::{Compressor, Decompressor}; -use crate::entry::EntryValue; +use crate::entry::{EntryValue, KindMeta}; use crate::kind::Kind; use crate::owner::Owner; use crate::stats::{IndexReadStats, IndexWriterStats}; @@ -91,19 +91,26 @@ pub struct IndexEntry { impl From for EntryValue { fn from(index_entry: IndexEntry) -> EntryValue { + let kind_meta = match index_entry.kind { + Kind::File => KindMeta::File { + size: index_entry.addrs.iter().map(|a| a.len).sum(), + }, + Kind::Symlink => KindMeta::Symlink { + // TODO: Should not be fatal + target: index_entry + .target + .expect("symlink entry should have a target"), + }, + Kind::Dir => KindMeta::Dir, + Kind::Unknown => KindMeta::Unknown, + }; EntryValue { apath: index_entry.apath, - kind: index_entry.kind, + kind_meta, mtime: OffsetDateTime::from_unix_seconds_and_nanos( index_entry.mtime, index_entry.mtime_nanos, ), - size: if index_entry.kind == Kind::File { - Some(index_entry.addrs.iter().map(|a| a.len).sum()) - } else { - None - }, - symlink_target: index_entry.target, unix_mode: index_entry.unix_mode, owner: index_entry.owner, } @@ -133,8 +140,8 @@ impl EntryTrait for IndexEntry { /// Target of the symlink, if this is a symlink. #[inline] - fn symlink_target(&self) -> &Option { - &self.target + fn symlink_target(&self) -> Option<&str> { + self.target.as_ref().map(String::as_str) } fn unix_mode(&self) -> UnixMode { @@ -158,7 +165,7 @@ impl IndexEntry { apath: source.apath().clone(), kind: source.kind(), addrs: Vec::new(), - target: source.symlink_target().clone(), + target: source.symlink_target().map(|t| t.to_owned()), mtime: mtime.unix_timestamp(), mtime_nanos: mtime.nanosecond(), unix_mode: source.unix_mode(), diff --git a/src/live_tree.rs b/src/live_tree.rs index 3fe61c41..38ed0972 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use tracing::{error, warn}; -use crate::entry::EntryValue; +use crate::entry::{EntryValue, KindMeta}; use crate::owner::Owner; use crate::stats::LiveTreeIterStats; use crate::unix_mode::UnixMode; @@ -79,30 +79,51 @@ impl tree::ReadTree for LiveTree { fn entry_from_fs_metadata( apath: Apath, + source_path: &Path, metadata: &fs::Metadata, - symlink_target: Option, -) -> EntryValue { - // TODO: Could we read the symlink target here, rather than in the caller? +) -> Result { let mtime = metadata .modified() .expect("Failed to get file mtime") .into(); - let size = if metadata.is_file() { - Some(metadata.len()) + let kind_meta = if metadata.is_file() { + KindMeta::File { + size: metadata.len(), + } + } else if metadata.is_dir() { + KindMeta::Dir + } else if metadata.is_symlink() { + let t = match source_path.read_link() { + Ok(t) => t, + Err(e) => { + error!("Failed to read target of symlink {source_path:?}: {e}"); + return Err(e.into()); + } + }; + let target = match t.into_os_string().into_string() { + Ok(t) => t, + Err(e) => { + error!("Failed to decode target of symlink {source_path:?}: {e:?}"); + return Err(Error::UnsupportedTargetEncoding { + path: source_path.to_owned(), + }); + } + }; + KindMeta::Symlink { target } } else { - None + return Err(Error::UnsupportedSourceKind { + path: source_path.to_owned(), + }); }; let owner = Owner::from(metadata); let unix_mode = UnixMode::from(metadata.permissions()); - EntryValue { + Ok(EntryValue { apath, - kind: metadata.file_type().into(), mtime, - symlink_target, - size, + kind_meta, unix_mode, owner, - } + }) } /// Recursive iterator of the contents of a live tree. @@ -138,13 +159,14 @@ impl Iter { /// Construct a new iter that will visit everything below this root path, /// subject to some exclusions fn new(root_path: &Path, subtree: Apath, exclude: Exclude) -> Result { - let start_metadata = fs::symlink_metadata(subtree.below(root_path))?; + let start_path = subtree.below(root_path); + let start_metadata = fs::symlink_metadata(&start_path)?; // Preload iter to return the root and then recurse into it. let entry_deque: VecDeque = [entry_from_fs_metadata( subtree.clone(), + &start_path, &start_metadata, - None, - )] + )?] .into(); // TODO: Consider the case where the root is not actually a directory? // Should that be supported? @@ -236,33 +258,18 @@ impl Iter { } }; - // TODO: Move this into entry_from_fs_metadata, once there's a - // global way for it to complain about errors. - let target: Option = if ft.is_symlink() { - let t = match dir_path.join(dir_entry.file_name()).read_link() { - Ok(t) => t, - Err(e) => { - error!("Failed to read target of symlink {child_apath:?}: {e}"); - continue; - } - }; - match t.into_os_string().into_string() { - Ok(t) => Some(t), - Err(e) => { - error!("Failed to decode target of symlink {child_apath:?}: {e:?}"); - continue; - } - } - } else { - None - }; if ft.is_dir() { subdir_apaths.push(child_apath.clone()); } - children.push(( - child_name.to_string(), - entry_from_fs_metadata(child_apath, &metadata, target), - )); + let child_path = dir_path.join(dir_entry.file_name()); + let entry = match entry_from_fs_metadata(child_apath, &child_path, &metadata) { + Ok(entry) => entry, + Err(err) => { + error!("Failed to build entry for {child_path:?}: {err:?}"); + continue; + } + }; + children.push((child_name.to_string(), entry)); } // To get the right overall tree ordering, any new subdirectories // discovered here should be visited together in apath order, but before From c660f8aebc4ce3bacd8b43891c1b71165838f71d Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 18:46:35 -0700 Subject: [PATCH 43/86] refactor --- src/archive.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index a8a556fe..961159b9 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -139,8 +139,8 @@ impl Archive { subtree: Apath, exclude: Exclude, ) -> Result> { - let stored_tree = self.open_stored_tree(band_selection)?; - stored_tree.iter_entries(subtree, exclude) + self.open_stored_tree(band_selection)? + .iter_entries(subtree, exclude) } /// Returns a vector of band ids, in sorted order from first to last. From ebcf35eb04b9e583060a8101103efe32522771a2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 May 2023 19:02:57 -0700 Subject: [PATCH 44/86] Add ls --json --- NEWS.md | 2 +- src/backup.rs | 2 +- src/bin/conserve.rs | 15 +++++++++++++-- src/index.rs | 2 +- tests/cli/ls.rs | 37 +++++++++++++++++++++++++++++++++++++ tests/cli/main.rs | 1 + 6 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 tests/cli/ls.rs diff --git a/NEWS.md b/NEWS.md index 62d5b502..898c5872 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,7 +16,7 @@ - `diff` output format has changed slightly to be the same as `backup`. -- New `diff --json`. +- New `diff --json` and `ls --json`. ## 23.1.1 diff --git a/src/backup.rs b/src/backup.rs index cd41e8c0..4724a8b0 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -269,7 +269,7 @@ impl BackupWriter { } fn copy_symlink(&mut self, source_entry: &EntryValue) -> Result> { - let target = source_entry.symlink_target().clone(); + let target = source_entry.symlink_target(); self.stats.symlinks += 1; assert!(target.is_some()); self.index_builder diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index ea97770d..d96060f6 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -153,9 +153,14 @@ enum Command { #[arg(long, short)] exclude: Vec, + #[arg(long, short = 'E')] exclude_from: Vec, + /// Print entries as json. + #[arg(long, short)] + json: bool, + /// Show permissions, owner, and group. #[arg(short = 'l')] long_listing: bool, @@ -408,6 +413,7 @@ impl Command { debug!("Created new archive in {archive:?}"); } Command::Ls { + json, stos, exclude, exclude_from, @@ -428,8 +434,13 @@ impl Command { .iter_entries(Apath::root(), exclude)?, ) }; - - show::show_entry_names(entry_iter, &mut stdout, *long_listing)?; + if *json { + for entry in entry_iter { + println!("{}", serde_json::ser::to_string(&entry)?); + } + } else { + show::show_entry_names(entry_iter, &mut stdout, *long_listing)?; + } } Command::Restore { archive, diff --git a/src/index.rs b/src/index.rs index e7f55a7b..a3b21af7 100644 --- a/src/index.rs +++ b/src/index.rs @@ -141,7 +141,7 @@ impl EntryTrait for IndexEntry { /// Target of the symlink, if this is a symlink. #[inline] fn symlink_target(&self) -> Option<&str> { - self.target.as_ref().map(String::as_str) + self.target.as_deref() } fn unix_mode(&self) -> UnixMode { diff --git a/tests/cli/ls.rs b/tests/cli/ls.rs new file mode 100644 index 00000000..e85c929e --- /dev/null +++ b/tests/cli/ls.rs @@ -0,0 +1,37 @@ +// Conserve backup system. +// Copyright 2023 Martin Pool. + +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Test `conserve ls`. + +use assert_cmd::prelude::*; +use indoc::indoc; +use pretty_assertions::assert_eq; + +use super::run_conserve; + +#[test] +fn ls_json() { + let cmd = run_conserve() + .args(["ls", "--json", "./testdata/archive/minimal/v0.6.17"]) + .assert() + .success(); + assert_eq!( + String::from_utf8_lossy(&cmd.get_output().stdout), + indoc! { r#" + {"apath":"/","kind":"Dir","mtime":"2020-06-16 00:15:23.0 +00:00:00","unix_mode":509,"user":"mbp","group":"mbp"} + {"apath":"/hello","kind":"File","size":12,"mtime":"2020-06-16 00:15:23.0 +00:00:00","unix_mode":436,"user":"mbp","group":"mbp"} + {"apath":"/subdir","kind":"Dir","mtime":"2020-06-16 00:15:23.0 +00:00:00","unix_mode":509,"user":"mbp","group":"mbp"} + {"apath":"/subdir/subfile","kind":"File","size":12,"mtime":"2020-06-16 00:15:23.0 +00:00:00","unix_mode":436,"user":"mbp","group":"mbp"} + "# } + ); +} diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 8f04fb86..8ba510dc 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -32,6 +32,7 @@ mod backup; mod delete; mod diff; mod exclude; +mod ls; mod trace; mod validate; mod versions; From ebd9deec697e773b80af70f999bcce1b825a2114 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 24 May 2023 07:19:48 -0700 Subject: [PATCH 45/86] Don't spam errors about unsupported file kinds --- src/live_tree.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/live_tree.rs b/src/live_tree.rs index 38ed0972..67593624 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -264,6 +264,11 @@ impl Iter { let child_path = dir_path.join(dir_entry.file_name()); let entry = match entry_from_fs_metadata(child_apath, &child_path, &metadata) { Ok(entry) => entry, + Err(Error::UnsupportedSourceKind { .. }) => { + // It's not too surprising that there would be fifos or sockets or files + // we don't support; don't log them. + continue; + } Err(err) => { error!("Failed to build entry for {child_path:?}: {err:?}"); continue; From 1a43d80cd15f77c344610f3a0bd0c2377f4b2496 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 24 May 2023 07:22:01 -0700 Subject: [PATCH 46/86] cargo update --- Cargo.lock | 504 +++++++++++++++++++++++++++-------------------------- 1 file changed, 256 insertions(+), 248 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb7687e2..9697423f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,6 +22,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.3.2" @@ -82,13 +91,14 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.8" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9834fcc22e0874394a010230586367d4a3e9f11b560f469262678547e1d2575e" +checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" dependencies = [ + "anstyle", "bstr", "doc-comment", - "predicates", + "predicates 3.0.3", "predicates-core", "predicates-tree", "wait-timeout", @@ -96,13 +106,14 @@ dependencies = [ [[package]] name = "assert_fs" -version = "1.0.10" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94b2a3f3786ff2996a98afbd6b4e5b7e890d685ccf67577f508ee2342c71cc9" +checksum = "f070617a68e5c2ed5d06ee8dd620ee18fb72b99f6c094bed34cf8ab07c875b48" dependencies = [ + "anstyle", "doc-comment", "globwalk", - "predicates", + "predicates 3.0.3", "predicates-core", "predicates-tree", "tempfile", @@ -164,9 +175,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.1.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b" +checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" dependencies = [ "memchr", "once_cell", @@ -176,9 +187,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -188,9 +199,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cachedir" @@ -203,9 +214,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -300,7 +311,7 @@ dependencies = [ "mutants", "nix", "nutmeg", - "predicates", + "predicates 2.1.5", "pretty_assertions", "proptest", "proptest-derive", @@ -348,9 +359,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -358,9 +369,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -369,22 +380,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.8.0", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -396,7 +407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote 1.0.27", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -409,7 +420,7 @@ dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", "rustc_version", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -438,9 +449,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "endian-type" @@ -448,17 +459,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - [[package]] name = "errno" version = "0.3.1" @@ -482,23 +482,23 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] [[package]] name = "filetime" -version = "0.2.19" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.42.0", + "redox_syscall 0.2.16", + "windows-sys 0.48.0", ] [[package]] @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -542,7 +542,7 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.20", "bstr", "fnv", "log", @@ -571,9 +571,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -634,9 +634,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", @@ -644,9 +644,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2d6f23ffea9d7e76c53eee25dfb67bcd8fde7f1198b0855350698c9f07c780" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" [[package]] name = "instant" @@ -659,12 +659,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.4" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ + "hermit-abi 0.3.1", "libc", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -675,7 +676,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix 0.37.3", + "rustix", "windows-sys 0.48.0", ] @@ -690,15 +691,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -711,15 +712,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] -name = "linux-raw-sys" -version = "0.1.4" +name = "libm" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" @@ -779,6 +780,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.20.1" @@ -787,7 +797,7 @@ checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ "ahash", "metrics-macros", - "portable-atomic", + "portable-atomic 0.3.20", ] [[package]] @@ -798,7 +808,7 @@ checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -807,7 +817,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.20", "crossbeam-epoch", "crossbeam-utils", "hashbrown", @@ -816,7 +826,7 @@ dependencies = [ "num_cpus", "ordered-float", "parking_lot", - "portable-atomic", + "portable-atomic 0.3.20", "quanta", "radix_trie", "sketches-ddsketch", @@ -846,7 +856,7 @@ dependencies = [ "bitflags", "cfg-if", "libc", - "memoffset", + "memoffset 0.7.1", "pin-utils", "static_assertions", ] @@ -880,6 +890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -914,9 +925,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "ordered-float" @@ -954,15 +965,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -985,9 +996,18 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" +checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" +dependencies = [ + "portable-atomic 1.3.2", +] + +[[package]] +name = "portable-atomic" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc59d1bcc64fc5d021d67521f818db868368028108d37f0e98d74e33f68297b5" [[package]] name = "ppv-lite86" @@ -1009,17 +1029,29 @@ dependencies = [ "regex", ] +[[package]] +name = "predicates" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" +dependencies = [ + "anstyle", + "difflib", + "itertools", + "predicates-core", +] + [[package]] name = "predicates-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" [[package]] name = "predicates-tree" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" dependencies = [ "predicates-core", "termtree", @@ -1057,22 +1089,22 @@ dependencies = [ [[package]] name = "proptest" -version = "1.0.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" +checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" dependencies = [ "bit-set", "bitflags", "byteorder", "lazy_static", "num-traits", - "quick-error 2.0.1", "rand", "rand_chacha", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.6.29", "rusty-fork", "tempfile", + "unarray", ] [[package]] @@ -1108,12 +1140,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quote" version = "0.6.13" @@ -1183,18 +1209,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.6.0" +version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ "bitflags", ] [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -1202,9 +1228,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -1227,15 +1253,24 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" -version = "1.7.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" dependencies = [ - "aho-corasick", + "aho-corasick 1.0.1", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -1244,23 +1279,20 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "regex-syntax" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rstest" @@ -1282,7 +1314,7 @@ dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", "rustc_version", - "syn 1.0.107", + "syn 1.0.109", "unicode-ident", ] @@ -1297,30 +1329,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.7" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", - "errno 0.2.8", + "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.1.4", - "windows-sys 0.42.0", -] - -[[package]] -name = "rustix" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2" -dependencies = [ - "bitflags", - "errno 0.3.1", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.45.0", + "linux-raw-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1330,16 +1348,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" dependencies = [ "fnv", - "quick-error 1.2.3", + "quick-error", "tempfile", "wait-timeout", ] [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -1358,35 +1376,35 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "semver" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1404,9 +1422,9 @@ dependencies = [ [[package]] name = "sketches-ddsketch" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" [[package]] name = "smallvec" @@ -1445,9 +1463,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", @@ -1467,52 +1485,51 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" dependencies = [ - "rustix 0.36.7", - "windows-sys 0.42.0", + "rustix", + "windows-sys 0.48.0", ] [[package]] name = "termtree" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", ] [[package]] @@ -1523,18 +1540,19 @@ checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "time" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "libc", @@ -1546,15 +1564,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -1570,9 +1588,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tracing" @@ -1599,20 +1617,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -1641,9 +1659,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -1663,9 +1681,9 @@ dependencies = [ [[package]] name = "tracing-test" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3d272c44878d2bbc9f4a20ad463724f03e19dbc667c6e84ac433ab7ffcc70b" +checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" dependencies = [ "lazy_static", "tracing-core", @@ -1675,26 +1693,32 @@ dependencies = [ [[package]] name = "tracing-test-macro" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744324b12d69a9fc1edea4b38b7b1311295b662d161ad5deac17bb1358224a08" +checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" dependencies = [ "lazy_static", "quote 1.0.27", - "syn 1.0.107", + "syn 1.0.109", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -1767,12 +1791,11 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -1790,9 +1813,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1800,24 +1823,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote 1.0.27", "wasm-bindgen-macro-support", @@ -1825,28 +1848,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.107", + "syn 2.0.16", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -1883,28 +1906,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", -] - [[package]] name = "windows-sys" version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.42.1", + "windows-targets 0.42.2", ] [[package]] @@ -1918,17 +1926,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -1948,9 +1956,9 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -1960,9 +1968,9 @@ checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -1972,9 +1980,9 @@ checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -1984,9 +1992,9 @@ checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -1996,9 +2004,9 @@ checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -2008,9 +2016,9 @@ checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -2020,9 +2028,9 @@ checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" From e9cea546f5be5094a483d17f387de6b59dc54615 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 24 May 2023 07:23:17 -0700 Subject: [PATCH 47/86] Update dependencies --- Cargo.lock | 96 +++++++++++++++++++++--------------------------------- Cargo.toml | 8 ++--- 2 files changed, 41 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9697423f..358c76c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,11 +4,11 @@ version = 3 [[package]] name = "ahash" -version = "0.7.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "getrandom", + "cfg-if", "once_cell", "version_check", ] @@ -98,7 +98,7 @@ dependencies = [ "anstyle", "bstr", "doc-comment", - "predicates 3.0.3", + "predicates", "predicates-core", "predicates-tree", "wait-timeout", @@ -113,7 +113,7 @@ dependencies = [ "anstyle", "doc-comment", "globwalk", - "predicates 3.0.3", + "predicates", "predicates-core", "predicates-tree", "tempfile", @@ -311,7 +311,7 @@ dependencies = [ "mutants", "nix", "nutmeg", - "predicates 2.1.5", + "predicates", "pretty_assertions", "proptest", "proptest-derive", @@ -533,7 +533,7 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -565,6 +565,12 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", ] @@ -639,14 +645,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "9f2cb48b81b1dc9f39676bf99f5499babfec7cd8fe14307f7b3d747208fb5690" [[package]] name = "instant" @@ -748,10 +754,10 @@ dependencies = [ ] [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" dependencies = [ "libc", ] @@ -791,42 +797,40 @@ dependencies = [ [[package]] name = "metrics" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" +checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" dependencies = [ "ahash", "metrics-macros", - "portable-atomic 0.3.20", + "portable-atomic", ] [[package]] name = "metrics-macros" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2 1.0.58", "quote 1.0.27", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] name = "metrics-util" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" +checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" dependencies = [ "aho-corasick 0.7.20", "crossbeam-epoch", "crossbeam-utils", - "hashbrown", + "hashbrown 0.13.2", "indexmap", "metrics", "num_cpus", "ordered-float", - "parking_lot", - "portable-atomic 0.3.20", "quanta", "radix_trie", "sketches-ddsketch", @@ -931,9 +935,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "ordered-float" -version = "2.10.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" dependencies = [ "num-traits", ] @@ -994,15 +998,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "portable-atomic" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" -dependencies = [ - "portable-atomic 1.3.2", -] - [[package]] name = "portable-atomic" version = "1.3.2" @@ -1015,20 +1010,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp", - "itertools", - "normalize-line-endings", - "predicates-core", - "regex", -] - [[package]] name = "predicates" version = "3.0.3" @@ -1037,8 +1018,11 @@ checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" dependencies = [ "anstyle", "difflib", + "float-cmp", "itertools", + "normalize-line-endings", "predicates-core", + "regex", ] [[package]] @@ -1120,16 +1104,16 @@ dependencies = [ [[package]] name = "quanta" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" +checksum = "8cc73c42f9314c4bdce450c77e6f09ecbddefbeddb1b5979ded332a3913ded33" dependencies = [ "crossbeam-utils", "libc", - "mach", + "mach2", "once_cell", "raw-cpuid", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -1799,12 +1783,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index f1f12f20..fde14847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,8 @@ globset = "0.4.5" hex = "0.4.2" itertools = "0.10" lazy_static = "1.4.0" -metrics = "0.20" -metrics-util = "0.14" +metrics = "0.21" +metrics-util = "0.15" mutants = "0.0.3" rayon = "1.3.0" readahead-iterator = "0.1.1" @@ -49,7 +49,7 @@ tracing = "0.1" tracing-appender = "0.2" unix_mode = "0.1" url = "2.2.2" -indoc = "1.0.8" +indoc = "2.0" [target.'cfg(unix)'.dependencies] users = "0.11" @@ -72,7 +72,7 @@ assert_cmd = "2.0" assert_fs = "1.0" cp_r = "0.5" dir-assert = "0.2" -predicates = "2" +predicates = "3" pretty_assertions = "1.0" proptest = "1.0" proptest-derive = "0.3" From 1bd15face492279772c96a43f913aae17e43304f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Wed, 24 May 2023 07:33:48 -0700 Subject: [PATCH 48/86] Prepare 23.5! --- Cargo.lock | 2 +- Cargo.toml | 2 +- NEWS.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 358c76c2..b7d5a288 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,7 +287,7 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "conserve" -version = "23.2.0" +version = "23.5.0" dependencies = [ "assert_cmd", "assert_fs", diff --git a/Cargo.toml b/Cargo.toml index fde14847..38d3f625 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-2.0" name = "conserve" readme = "README.md" repository = "https://github.com/sourcefrog/conserve/" -version = "23.2.0" +version = "23.5.0" rust-version = "1.63" [[bin]] diff --git a/NEWS.md b/NEWS.md index 898c5872..d5b97a0f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # Conserve release history -## Unreleased +## 23.5.0 - Better progress bars for various operations including `validate`. From 55e23258c763603943bea23901aff759a76cff19 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 23 Jul 2023 09:11:58 -0700 Subject: [PATCH 49/86] CI variation for install --locked --- .github/workflows/install.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 32a2d027..a40545a8 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -7,8 +7,11 @@ on: jobs: cargo-install: + strategy: + matrix: + locked: ["", "--locked"] runs-on: ubuntu-latest steps: - name: cargo-install run: | - cargo install cargo-mutants + cargo install cargo-mutants ${{ matrix.locked }} From 086a65c2a11a4fda2336c96dbe3a5729e80015dc Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sat, 19 Aug 2023 16:21:05 -0700 Subject: [PATCH 50/86] Don't really need serializable errors --- src/errors.rs | 143 ++++++++------------------------------------------ 1 file changed, 21 insertions(+), 122 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index f8df0977..3dcc58e2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -17,35 +17,14 @@ use std::borrow::Cow; use std::io; use std::path::PathBuf; -use serde::{self, ser::SerializeStruct, Serialize, Serializer}; use thiserror::Error; use crate::blockdir::Address; use crate::*; -fn serialize_io_error(err: &io::Error, s: S) -> std::result::Result -where - S: Serializer, -{ - let mut t = s.serialize_struct("io::Error", 2)?; - t.serialize_field("os_error", &err.raw_os_error())?; - t.serialize_field("message", &err.to_string())?; - t.end() -} - -fn serialize_generic_error(err: &E, s: S) -> std::result::Result -where - E: std::error::Error, - S: Serializer, -{ - let mut t = s.serialize_struct("Error", 1)?; - t.serialize_field("message", &err.to_string())?; - t.end() -} - /// Conserve specific error. #[non_exhaustive] -#[derive(Debug, Error, Serialize)] +#[derive(Debug, Error)] pub enum Error { #[error("Block file {hash:?} corrupt; actual hash {actual_hash:?}")] BlockCorrupt { hash: String, actual_hash: String }, @@ -64,36 +43,22 @@ pub enum Error { }, #[error("Failed to write block {hash:?}")] - WriteBlock { - hash: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + WriteBlock { hash: String, source: io::Error }, #[error("Failed to read block {hash:?}")] - ReadBlock { - hash: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ReadBlock { hash: String, source: io::Error }, #[error("Block {block_hash} is missing")] BlockMissing { block_hash: BlockHash }, #[error("Failed to list block files")] - ListBlocks { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ListBlocks { source: io::Error }, #[error("Not a Conserve archive")] NotAnArchive {}, #[error("Failed to read archive header")] - ReadArchiveHeader { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ReadArchiveHeader { source: io::Error }, #[error( "Archive version {:?} is not supported by Conserve {}", @@ -131,25 +96,16 @@ pub enum Error { InvalidVersion { version: String }, #[error("Failed to create band")] - CreateBand { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + CreateBand { source: io::Error }, #[error("Band {band_id} head file missing")] BandHeadMissing { band_id: BandId }, #[error("Failed to create block directory")] - CreateBlockDir { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + CreateBlockDir { source: io::Error }, #[error("Failed to create archive directory")] - CreateArchiveDirectory { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + CreateArchiveDirectory { source: io::Error }, #[error("Band {} is incomplete", band_id)] BandIncomplete { band_id: BandId }, @@ -172,77 +128,47 @@ pub enum Error { #[error(transparent)] ParseGlob { #[from] - #[serde(serialize_with = "serialize_generic_error")] source: globset::Error, }, #[error("Failed to write index hunk {:?}", path)] - WriteIndex { - path: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + WriteIndex { path: String, source: io::Error }, #[error("Failed to read index hunk {:?}", path)] - ReadIndex { - path: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ReadIndex { path: String, source: io::Error }, #[error("Failed to serialize index")] - SerializeIndex { - #[serde(serialize_with = "serialize_generic_error")] - source: serde_json::Error, - }, + SerializeIndex { source: serde_json::Error }, #[error("Failed to deserialize index hunk {:?}", path)] DeserializeIndex { path: String, - #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Failed to write metadata file {:?}", path)] - WriteMetadata { - path: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + WriteMetadata { path: String, source: io::Error }, #[error("Failed to deserialize json from {:?}", path)] DeserializeJson { path: PathBuf, - #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Failed to serialize json to {:?}", path)] SerializeJson { path: String, - #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, #[error("Metadata file not found: {:?}", path)] - MetadataNotFound { - path: String, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + MetadataNotFound { path: String, source: io::Error }, #[error("Failed to list bands")] - ListBands { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ListBands { source: io::Error }, #[error("Failed to read source file {:?}", path)] - ReadSourceFile { - path: PathBuf, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ReadSourceFile { path: PathBuf, source: io::Error }, #[error("Unsupported source file kind: {path:?}")] UnsupportedSourceKind { path: PathBuf }, @@ -251,39 +177,19 @@ pub enum Error { UnsupportedTargetEncoding { path: PathBuf }, #[error("Failed to read source tree {:?}", path)] - ListSourceTree { - path: PathBuf, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + ListSourceTree { path: PathBuf, source: io::Error }, #[error("Failed to store file {:?}", apath)] - StoreFile { - apath: Apath, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + StoreFile { apath: Apath, source: io::Error }, #[error("Failed to restore {:?}", path)] - Restore { - path: PathBuf, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + Restore { path: PathBuf, source: io::Error }, #[error("Failed to restore modification time on {:?}", path)] - RestoreModificationTime { - path: PathBuf, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + RestoreModificationTime { path: PathBuf, source: io::Error }, #[error("Failed to delete band {}", band_id)] - BandDeletion { - band_id: BandId, - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - }, + BandDeletion { band_id: BandId, source: io::Error }, #[error("Unsupported URL scheme {:?}", scheme)] UrlScheme { scheme: String }, @@ -291,7 +197,6 @@ pub enum Error { #[error("Failed to serialize object")] SerializeError { #[from] - #[serde(serialize_with = "serialize_generic_error")] source: serde_json::Error, }, @@ -302,21 +207,15 @@ pub enum Error { #[error(transparent)] IOError { #[from] - #[serde(serialize_with = "serialize_io_error")] source: io::Error, }, #[error("Failed to set owner of {path:?}")] - SetOwner { - #[serde(serialize_with = "serialize_io_error")] - source: io::Error, - path: PathBuf, - }, + SetOwner { source: io::Error, path: PathBuf }, #[error(transparent)] SnapCompressionError { // TODO: Maybe say in which file, etc. - #[serde(serialize_with = "serialize_generic_error")] #[from] source: snap::Error, }, From 4af6bf72d1dbe23f6ac3099d6ef701c6e7b03830 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 20 Aug 2023 21:03:17 -0700 Subject: [PATCH 51/86] Clippy: remove unnecessary hashes around strings --- tests/api/transport.rs | 2 +- tests/cli/delete.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/api/transport.rs b/tests/api/transport.rs index f2a05660..92ba92e8 100644 --- a/tests/api/transport.rs +++ b/tests/api/transport.rs @@ -51,7 +51,7 @@ fn parse_location_urls() { assert_eq!(parsed_scheme("/backup/repo.c6"), "file"); assert_eq!(parsed_scheme("../backup/repo.c6"), "file"); assert_eq!(parsed_scheme("c:/backup/repo"), "file"); - assert_eq!(parsed_scheme(r#"c:\backup\repo\"#), "file"); + assert_eq!(parsed_scheme(r"c:\backup\repo\"), "file"); } #[test] diff --git a/tests/cli/delete.rs b/tests/cli/delete.rs index a9a6fc83..ab21574b 100644 --- a/tests/cli/delete.rs +++ b/tests/cli/delete.rs @@ -127,7 +127,7 @@ fn delete_nonexistent_band() { "ERROR conserve: Failed to delete band b0000", )) .stderr( - predicate::str::is_match(r#"caused by: (File not found.|No such file or directory|The system cannot find the file specified\.) \(os error \d+\)"#) + predicate::str::is_match(r"caused by: (File not found.|No such file or directory|The system cannot find the file specified\.) \(os error \d+\)") .unwrap(), ) .failure(); From e5eeac08725f10143c2f93db05b3f65b64aa1bce Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 22 Aug 2023 07:12:34 -0700 Subject: [PATCH 52/86] Clippy --- src/owner/unix.rs | 2 +- src/progress.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/owner/unix.rs b/src/owner/unix.rs index b1241d27..5f768e22 100644 --- a/src/owner/unix.rs +++ b/src/owner/unix.rs @@ -62,7 +62,7 @@ pub(crate) fn set_owner(owner: &Owner, path: &Path) -> Result<()> { // TODO: use `std::os::unix::fs::chown(path, uid, gid)?;` once stable match unistd::chown(path, uid_opt, gid_opt) { Ok(()) => Ok(()), - Err(errno) if errno == Errno::EPERM => { + Err(Errno::EPERM) => { // If the restore is not run as root (or with special capabilities) // then we probably can't set ownership, and there's no point // complaining diff --git a/src/progress.rs b/src/progress.rs index 41175174..db0ae2c3 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -38,7 +38,7 @@ impl ProgressImpl { *IMPL.write().expect("locked progress impl") = self } - fn remove_bar(&mut self, task: &mut Bar) { + fn remove_bar(&mut self, task: &Bar) { match self { ProgressImpl::Null => (), ProgressImpl::Terminal => term::remove_bar(task.bar_id), From b7e653c12d2f8bef874d90c059bb1305ae9caea5 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 22 Aug 2023 07:17:40 -0700 Subject: [PATCH 53/86] Tentative migration towards anyhow --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/bin/conserve.rs | 21 +++++++++------------ tests/cli/delete.rs | 7 ++++++- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7d5a288..5b346484 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + [[package]] name = "arrayvec" version = "0.4.12" @@ -289,6 +295,7 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" name = "conserve" version = "23.5.0" dependencies = [ + "anyhow", "assert_cmd", "assert_fs", "assert_matches", diff --git a/Cargo.toml b/Cargo.toml index 38d3f625..9948a396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ doc = false name = "conserve" [dependencies] +anyhow = "1.0" assert_matches = "1.5.0" blake2-rfc = "0.2.18" bytes = "1.1.0" diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index d96060f6..05e1d589 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -14,7 +14,6 @@ //! Command-line entry point for Conserve backups. use std::cell::RefCell; -use std::error::Error; use std::fs::OpenOptions; use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; @@ -291,7 +290,7 @@ impl std::process::Termination for ExitCode { } impl Command { - fn run(&self) -> Result { + fn run(&self) -> anyhow::Result { let mut stdout = std::io::stdout(); match self { Command::Backup { @@ -532,10 +531,13 @@ impl Command { } } -fn stored_tree_from_opt(archive_location: &str, backup: &Option) -> Result { +fn stored_tree_from_opt( + archive_location: &str, + backup: &Option, +) -> anyhow::Result { let archive = Archive::open(open_transport(archive_location)?)?; let policy = band_selection_policy_from_opt(backup); - archive.open_stored_tree(policy) + Ok(archive.open_stored_tree(policy)?) } fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolicy { @@ -550,7 +552,7 @@ fn make_change_callback<'a>( print_changes: bool, ls_long: bool, changes_json: &Option<&Path>, -) -> Result>> { +) -> anyhow::Result>> { if !print_changes && !ls_long && changes_json.is_none() { return Ok(None); }; @@ -594,7 +596,7 @@ fn make_change_callback<'a>( }))) } -fn main() -> Result { +fn main() -> anyhow::Result { let args = Args::parse(); let start_time = Instant::now(); if !args.no_progress { @@ -619,12 +621,7 @@ fn main() -> Result { } match result { Err(err) => { - error!("{err}"); - let mut err: &dyn Error = &err; - while let Some(source) = err.source() { - error!("caused by: {source}"); - err = source; - } + error!("{err:#}"); debug!(error_count, warn_count,); Ok(ExitCode::Failure) } diff --git a/tests/cli/delete.rs b/tests/cli/delete.rs index ab21574b..1c4458fe 100644 --- a/tests/cli/delete.rs +++ b/tests/cli/delete.rs @@ -127,7 +127,12 @@ fn delete_nonexistent_band() { "ERROR conserve: Failed to delete band b0000", )) .stderr( - predicate::str::is_match(r"caused by: (File not found.|No such file or directory|The system cannot find the file specified\.) \(os error \d+\)") + predicate::str::is_match( + "(File not found\\.\ + |No such file or directory\ + |The system cannot find the file specified\\.) \ + \\(os error \\d+\\)", + ) .unwrap(), ) .failure(); From 21c6e1acacb6d86d2ba52d25c9a4a45dd0f14dd2 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 22 Aug 2023 07:23:50 -0700 Subject: [PATCH 54/86] Convert remove_dir, remove_dir_all and callers to Anyhow --- src/archive.rs | 2 +- src/band.rs | 8 +++----- src/transport.rs | 11 +++++------ src/transport/local.rs | 11 ++++++----- tests/api/gc.rs | 2 +- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 961159b9..ab86bf96 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -250,7 +250,7 @@ impl Archive { &self, delete_band_ids: &[BandId], options: &DeleteOptions, - ) -> Result { + ) -> anyhow::Result { let mut stats = DeleteStats::default(); let start = Instant::now(); diff --git a/src/band.rs b/src/band.rs index 90525575..ed340855 100644 --- a/src/band.rs +++ b/src/band.rs @@ -23,6 +23,7 @@ use std::borrow::Cow; +use anyhow::Context; use itertools::Itertools; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -214,15 +215,12 @@ impl Band { } /// Delete a band. - pub fn delete(archive: &Archive, band_id: &BandId) -> Result<()> { + pub fn delete(archive: &Archive, band_id: &BandId) -> anyhow::Result<()> { // TODO: Count how many files were deleted, and the total size? archive .transport() .remove_dir_all(&band_id.to_string()) - .map_err(|source| Error::BandDeletion { - band_id: band_id.clone(), - source, - }) + .with_context(|| format!("Failed to delete band {band_id}")) } pub fn is_closed(&self) -> Result { diff --git a/src/transport.rs b/src/transport.rs index 1f6bb745..25e27350 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -17,6 +17,7 @@ use std::io; use std::path::Path; +use anyhow::bail; use bytes::Bytes; use url::Url; @@ -28,7 +29,7 @@ use local::LocalTransport; /// Open a `Transport` to access a local directory. /// /// `s` may be a local path or a URL. -pub fn open_transport(s: &str) -> Result> { +pub fn open_transport(s: &str) -> anyhow::Result> { if let Ok(url) = Url::parse(s) { match url.scheme() { "file" => Ok(Box::new(LocalTransport::new( @@ -38,9 +39,7 @@ pub fn open_transport(s: &str) -> Result> { // Probably a Windows path with drive letter, like "c:/thing", not actually a URL. Ok(Box::new(LocalTransport::new(Path::new(s)))) } - other => Err(Error::UrlScheme { - scheme: other.to_owned(), - }), + other => bail!("Unsupported URL scheme {other:?}"), } } else { Ok(Box::new(LocalTransport::new(Path::new(s)))) @@ -136,10 +135,10 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn remove_file(&self, relpath: &str) -> io::Result<()>; /// Delete an empty directory. - fn remove_dir(&self, relpath: &str) -> io::Result<()>; + fn remove_dir(&self, relpath: &str) -> anyhow::Result<()>; /// Delete a directory and all its contents. - fn remove_dir_all(&self, relpath: &str) -> io::Result<()>; + fn remove_dir_all(&self, relpath: &str) -> anyhow::Result<()>; /// Make a new transport addressing a subdirectory. fn sub_transport(&self, relpath: &str) -> Box; diff --git a/src/transport/local.rs b/src/transport/local.rs index 3df800f5..c58f6ef0 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -18,6 +18,7 @@ use std::io; use std::io::prelude::*; use std::path::{Path, PathBuf}; +use anyhow::Context; use bytes::Bytes; use metrics::{counter, increment_counter}; @@ -122,12 +123,12 @@ impl Transport for LocalTransport { std::fs::remove_file(self.full_path(relpath)) } - fn remove_dir(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_dir(self.full_path(relpath)) + fn remove_dir(&self, relpath: &str) -> anyhow::Result<()> { + std::fs::remove_dir(self.full_path(relpath)).context("Remove directory") } - fn remove_dir_all(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_dir_all(self.full_path(relpath)) + fn remove_dir_all(&self, relpath: &str) -> anyhow::Result<()> { + std::fs::remove_dir_all(self.full_path(relpath)).context("Remove directory tree") } fn sub_transport(&self, relpath: &str) -> Box { @@ -305,7 +306,7 @@ mod test { } #[test] - fn remove_dir_all() -> std::io::Result<()> { + fn remove_dir_all() -> anyhow::Result<()> { let temp = assert_fs::TempDir::new().unwrap(); let transport = LocalTransport::new(temp.path()); diff --git a/tests/api/gc.rs b/tests/api/gc.rs index 3a59940e..3e6ab5f8 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -90,7 +90,7 @@ fn unreferenced_blocks() { } #[test] -fn backup_prevented_by_gc_lock() -> Result<()> { +fn backup_prevented_by_gc_lock() -> anyhow::Result<()> { let archive = ScratchArchive::new(); let tf = TreeFixture::new(); tf.create_file("hello"); From 2f72ccb22e004fc1ca7e43408a184d5a58a78749 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 22 Aug 2023 08:04:36 -0700 Subject: [PATCH 55/86] anyhow contaigion --- src/archive.rs | 24 +++++++++++++----------- src/backup.rs | 31 ++++++++++++++++++------------- src/band.rs | 6 ++---- src/bin/conserve.rs | 2 +- src/blockdir.rs | 15 +++++++-------- src/diff.rs | 2 +- src/gc_lock.rs | 30 ++++++++++++++++-------------- src/index.rs | 16 ++++++---------- src/live_tree.rs | 6 +++--- src/restore.rs | 14 ++++---------- src/show.rs | 2 +- src/stitch.rs | 2 +- src/stored_tree.rs | 20 +++++++++++--------- src/transport.rs | 20 ++++---------------- src/transport/local.rs | 22 ++++++++++++++-------- src/tree.rs | 6 +++--- src/validate.rs | 4 ++-- tests/api/damaged.rs | 4 ++-- tests/api/gc.rs | 8 ++++---- tests/api/restore.rs | 5 ++++- tests/cli/main.rs | 8 +++++--- 21 files changed, 122 insertions(+), 125 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index ab86bf96..a7b004c5 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; +use anyhow::anyhow; use itertools::Itertools; use rayon::prelude::*; use serde::{Deserialize, Serialize}; @@ -120,16 +121,14 @@ impl Archive { &self.block_dir } - pub fn band_exists(&self, band_id: &BandId) -> Result { + pub fn band_exists(&self, band_id: &BandId) -> anyhow::Result { self.transport .is_file(&format!("{}/{}", band_id, crate::BAND_HEAD_FILENAME)) - .map_err(Error::from) } - pub fn band_is_closed(&self, band_id: &BandId) -> Result { + pub fn band_is_closed(&self, band_id: &BandId) -> anyhow::Result { self.transport .is_file(&format!("{}/{}", band_id, crate::BAND_TAIL_FILENAME)) - .map_err(Error::from) } /// Return an iterator of entries in a selected version. @@ -138,7 +137,7 @@ impl Archive { band_selection: BandSelectionPolicy, subtree: Apath, exclude: Exclude, - ) -> Result> { + ) -> anyhow::Result> { self.open_stored_tree(band_selection)? .iter_entries(subtree, exclude) } @@ -154,18 +153,21 @@ impl Archive { self.transport.as_ref() } - pub fn resolve_band_id(&self, band_selection: BandSelectionPolicy) -> Result { + pub fn resolve_band_id(&self, band_selection: BandSelectionPolicy) -> anyhow::Result { match band_selection { BandSelectionPolicy::LatestClosed => self .last_complete_band()? .map(|band| band.id().clone()) - .ok_or(Error::ArchiveEmpty), + .ok_or(anyhow!("Archive has no complete bands")), BandSelectionPolicy::Specified(band_id) => Ok(band_id), - BandSelectionPolicy::Latest => self.last_band_id()?.ok_or(Error::ArchiveEmpty), + BandSelectionPolicy::Latest => self.last_band_id()?.ok_or(anyhow!("Archive is empty")), } } - pub fn open_stored_tree(&self, band_selection: BandSelectionPolicy) -> Result { + pub fn open_stored_tree( + &self, + band_selection: BandSelectionPolicy, + ) -> anyhow::Result { StoredTree::open(self, &self.resolve_band_id(band_selection)?) } @@ -192,7 +194,7 @@ impl Archive { } /// Return the last completely-written band id, if any. - pub fn last_complete_band(&self) -> Result> { + pub fn last_complete_band(&self) -> anyhow::Result> { for id in self.list_band_ids()?.iter().rev() { let b = Band::open(self, id)?; if b.is_closed()? { @@ -336,7 +338,7 @@ impl Archive { /// If problems are found, they are emitted as `warn` or `error` level /// tracing messages. This function only returns an error if validation /// stops due to a fatal error. - pub fn validate(&self, options: &ValidateOptions) -> Result<()> { + pub fn validate(&self, options: &ValidateOptions) -> anyhow::Result<()> { self.validate_archive_dir()?; debug!("List bands..."); diff --git a/src/backup.rs b/src/backup.rs index 4724a8b0..e57392ce 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -20,6 +20,7 @@ use std::io::prelude::*; use std::path::Path; use std::time::{Duration, Instant}; +use anyhow::bail; use derive_more::{Add, AddAssign}; use itertools::Itertools; use tracing::error; @@ -77,7 +78,7 @@ pub fn backup( archive: &Archive, source_path: &Path, options: &BackupOptions, -) -> Result { +) -> anyhow::Result { let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; let mut stats = BackupStats::default(); @@ -154,9 +155,9 @@ impl BackupWriter { /// Create a new BackupWriter. /// /// This currently makes a new top-level band. - pub fn begin(archive: &Archive) -> Result { + pub fn begin(archive: &Archive) -> anyhow::Result { if gc_lock::GarbageCollectionLock::is_locked(archive)? { - return Err(Error::GarbageCollectionLockHeld); + bail!("Archive is locked for garbage collection"); } let basis_index = IterStitchedIndexHunks::new(archive, archive.last_band_id()?) .iter_entries(Apath::root(), Exclude::nothing()); @@ -174,7 +175,7 @@ impl BackupWriter { }) } - fn finish(self) -> Result { + fn finish(self) -> anyhow::Result { let index_builder_stats = self.index_builder.finish()?; self.band.close(index_builder_stats.index_hunks as u64)?; Ok(BackupStats { @@ -184,7 +185,7 @@ impl BackupWriter { } /// Write out any pending data blocks, and then the pending index entries. - fn flush_group(&mut self) -> Result<()> { + fn flush_group(&mut self) -> anyhow::Result<()> { let (stats, mut entries) = self.file_combiner.drain()?; self.stats += stats; self.index_builder.append_entries(&mut entries); @@ -196,7 +197,11 @@ impl BackupWriter { /// Return an indication of whether it changed (if it's a file), or /// None for non-plain-file types where that information is not currently /// calculated. - fn copy_entry(&mut self, entry: &EntryValue, source: &LiveTree) -> Result> { + fn copy_entry( + &mut self, + entry: &EntryValue, + source: &LiveTree, + ) -> anyhow::Result> { // TODO: Emit deletions for entries in the basis not present in the source. match entry.kind() { Kind::Dir => self.copy_dir(entry), @@ -212,7 +217,7 @@ impl BackupWriter { } } - fn copy_dir(&mut self, source_entry: &EntryValue) -> Result> { + fn copy_dir(&mut self, source_entry: &EntryValue) -> anyhow::Result> { self.stats.directories += 1; self.index_builder .push_entry(IndexEntry::metadata_from(source_entry)); @@ -224,7 +229,7 @@ impl BackupWriter { &mut self, source_entry: &EntryValue, from_tree: &LiveTree, - ) -> Result> { + ) -> anyhow::Result> { self.stats.files += 1; let apath = source_entry.apath(); let result; @@ -268,7 +273,7 @@ impl BackupWriter { Ok(result) } - fn copy_symlink(&mut self, source_entry: &EntryValue) -> Result> { + fn copy_symlink(&mut self, source_entry: &EntryValue) -> anyhow::Result> { let target = source_entry.symlink_target(); self.stats.symlinks += 1; assert!(target.is_some()); @@ -284,7 +289,7 @@ fn store_file_content( from_file: &mut dyn Read, block_dir: &mut BlockDir, stats: &mut BackupStats, -) -> Result> { +) -> anyhow::Result> { let mut buffer = Vec::new(); let mut addresses = Vec::
::with_capacity(1); loop { @@ -353,7 +358,7 @@ impl FileCombiner { /// Flush any pending files, and return accumulated file entries and stats. /// The FileCombiner is then empty and ready for reuse. - fn drain(&mut self) -> Result<(BackupStats, Vec)> { + fn drain(&mut self) -> anyhow::Result<(BackupStats, Vec)> { self.flush()?; debug_assert!(self.queue.is_empty()); debug_assert!(self.buf.is_empty()); @@ -369,7 +374,7 @@ impl FileCombiner { /// /// After this call the FileCombiner is empty and can be reused for more files into a new /// block. - fn flush(&mut self) -> Result<()> { + fn flush(&mut self) -> anyhow::Result<()> { if self.queue.is_empty() { debug_assert!(self.buf.is_empty()); return Ok(()); @@ -394,7 +399,7 @@ impl FileCombiner { /// Add the contents of a small file into this combiner. /// /// `entry` should be an IndexEntry that's complete apart from the block addresses. - fn push_file(&mut self, entry: &EntryValue, from_file: &mut dyn Read) -> Result<()> { + fn push_file(&mut self, entry: &EntryValue, from_file: &mut dyn Read) -> anyhow::Result<()> { let start = self.buf.len(); let expected_len: usize = entry .size() diff --git a/src/band.rs b/src/band.rs index ed340855..a71e1f45 100644 --- a/src/band.rs +++ b/src/band.rs @@ -223,10 +223,8 @@ impl Band { .with_context(|| format!("Failed to delete band {band_id}")) } - pub fn is_closed(&self) -> Result { - self.transport - .is_file(BAND_TAIL_FILENAME) - .map_err(Error::from) + pub fn is_closed(&self) -> anyhow::Result { + self.transport.is_file(BAND_TAIL_FILENAME) } pub fn id(&self) -> &BandId { diff --git a/src/bin/conserve.rs b/src/bin/conserve.rs index 05e1d589..654c16c1 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve.rs @@ -537,7 +537,7 @@ fn stored_tree_from_opt( ) -> anyhow::Result { let archive = Archive::open(open_transport(archive_location)?)?; let policy = band_selection_policy_from_opt(backup); - Ok(archive.open_stored_tree(policy)?) + archive.open_stored_tree(policy) } fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolicy { diff --git a/src/blockdir.rs b/src/blockdir.rs index e7806f91..943efdb1 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -30,6 +30,7 @@ use std::sync::Arc; use std::time::Instant; use ::metrics::{counter, histogram, increment_counter}; +use anyhow::Context; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; use rayon::prelude::*; @@ -148,7 +149,7 @@ impl BlockDir { &mut self, block_data: &[u8], stats: &mut BackupStats, - ) -> Result { + ) -> anyhow::Result { let hash = self.hash_bytes(block_data); let len = block_data.len() as u64; if self.contains(&hash)? { @@ -168,14 +169,12 @@ impl BlockDir { } /// True if the named block is present in this directory. - pub fn contains(&self, hash: &BlockHash) -> Result { - self.transport - .is_file(&block_relpath(hash)) - .map_err(Error::from) + pub fn contains(&self, hash: &BlockHash) -> anyhow::Result { + self.transport.is_file(&block_relpath(hash)) } /// Returns the compressed on-disk size of a block. - pub fn compressed_size(&self, hash: &BlockHash) -> Result { + pub fn compressed_size(&self, hash: &BlockHash) -> anyhow::Result { Ok(self.transport.metadata(&block_relpath(hash))?.len) } @@ -202,10 +201,10 @@ impl BlockDir { } } - pub fn delete_block(&self, hash: &BlockHash) -> Result<()> { + pub fn delete_block(&self, hash: &BlockHash) -> anyhow::Result<()> { self.transport .remove_file(&block_relpath(hash)) - .map_err(Error::from) + .with_context(|| format!("Failed to delete block {hash}")) } /// Return an iterator of block subdirectories, in arbitrary order. diff --git a/src/diff.rs b/src/diff.rs index ec5c08a8..c97b6002 100644 --- a/src/diff.rs +++ b/src/diff.rs @@ -41,7 +41,7 @@ pub fn diff( st: &StoredTree, lt: &LiveTree, options: &DiffOptions, -) -> Result> { +) -> anyhow::Result> { let readahead = 1000; let include_unchanged: bool = options.include_unchanged; // Copy out to avoid lifetime problems in the callback let ait = st diff --git a/src/gc_lock.rs b/src/gc_lock.rs index ee96cdde..b89e840a 100644 --- a/src/gc_lock.rs +++ b/src/gc_lock.rs @@ -31,6 +31,8 @@ //! delete but before starting to actually delete them, we check that no //! new bands have been created. +use anyhow::{bail, Context}; + use crate::*; pub static GC_LOCK: &str = "GC_LOCK"; @@ -52,16 +54,16 @@ impl GarbageCollectionLock { /// /// Returns `Err(Error::DeleteWithIncompleteBackup)` if the last /// backup is incomplete. - pub fn new(archive: &Archive) -> Result { + pub fn new(archive: &Archive) -> anyhow::Result { let archive = archive.clone(); let band_id = archive.last_band_id()?; if let Some(band_id) = band_id.clone() { if !archive.band_is_closed(&band_id)? { - return Err(Error::DeleteWithIncompleteBackup { band_id }); + bail!("Can't delete blocks because the last band ({band_id}) is incomplete and may be in use" ); } } if archive.transport().is_file(GC_LOCK).unwrap_or(true) { - return Err(Error::GarbageCollectionLockHeld {}); + bail!("GC lock held"); } archive.transport().write_file(GC_LOCK, b"{}\n")?; Ok(GarbageCollectionLock { archive, band_id }) @@ -71,26 +73,29 @@ impl GarbageCollectionLock { /// /// Use this only if you're confident that the process owning the lock /// has terminated and the lock is stale. - pub fn break_lock(archive: &Archive) -> Result { + pub fn break_lock(archive: &Archive) -> anyhow::Result { if GarbageCollectionLock::is_locked(archive)? { - archive.transport().remove_file(GC_LOCK)?; + archive + .transport() + .remove_file(GC_LOCK) + .context("Failed to remove GC_LOCK")?; } - GarbageCollectionLock::new(archive) + GarbageCollectionLock::new(archive).context("Failed to take GC lock") } /// Returns true if the archive is currently locked by a gc process. - pub fn is_locked(archive: &Archive) -> Result { - archive.transport().is_file(GC_LOCK).map_err(Error::from) + pub fn is_locked(archive: &Archive) -> anyhow::Result { + archive.transport().is_file(GC_LOCK) } /// Check that no new versions have been created in this archive since /// the guard was created. - pub fn check(&self) -> Result<()> { + pub fn check(&self) -> anyhow::Result<()> { let current_last_band_id = self.archive.last_band_id()?; if self.band_id == current_last_band_id { Ok(()) } else { - Err(Error::DeleteWithConcurrentActivity) + bail!("A backup was created concurrently with GC: CHECK THE ARCHIVE INTEGRITY") } } } @@ -161,10 +166,7 @@ mod test { let _lock1 = GarbageCollectionLock::new(&archive).unwrap(); // Should not be able to create a second lock while one gc is running. let lock2_result = GarbageCollectionLock::new(&archive); - match lock2_result { - Err(Error::GarbageCollectionLockHeld) => (), - other => panic!("unexpected result {other:?}"), - }; + assert_eq!(lock2_result.unwrap_err().to_string(), "GC lock held"); } #[test] diff --git a/src/index.rs b/src/index.rs index a3b21af7..a710d9af 100644 --- a/src/index.rs +++ b/src/index.rs @@ -214,7 +214,7 @@ impl IndexWriter { } /// Finish the last hunk of this index, and return the stats. - pub fn finish(mut self) -> Result { + pub fn finish(mut self) -> anyhow::Result { self.finish_hunk()?; Ok(self.stats) } @@ -238,7 +238,7 @@ impl IndexWriter { /// This writes all the currently queued entries into a new index file /// in the band directory, and then clears the buffer to start receiving /// entries for the next hunk. - pub fn finish_hunk(&mut self) -> Result<()> { + pub fn finish_hunk(&mut self) -> anyhow::Result<()> { if self.entries.is_empty() { return Ok(()); } @@ -306,17 +306,13 @@ impl IndexRead { } /// Return the (1-based) number of index hunks in an index directory. - pub fn count_hunks(&self) -> Result { + pub fn count_hunks(&self) -> anyhow::Result { // TODO: Might be faster to list the directory than to probe for all of them. // TODO: Perhaps, list the directories and cope cleanly with // one hunk being missing. for i in 0.. { let path = hunk_relpath(i); - if !self - .transport - .is_file(&path) - .map_err(|source| Error::ReadIndex { source, path })? - { + if !self.transport.is_file(&path)? { // If hunk 1 is missing, 1 hunks exists. return Ok(i); } @@ -324,7 +320,7 @@ impl IndexRead { unreachable!(); } - pub fn estimate_entry_count(&self) -> Result { + pub fn estimate_entry_count(&self) -> anyhow::Result { Ok(u64::from(self.count_hunks()?) * (MAX_ENTRIES_PER_HUNK as u64)) } @@ -830,7 +826,7 @@ mod tests { /// /// https://github.com/sourcefrog/conserve/issues/95 #[test] - fn no_final_empty_hunk() -> Result<()> { + fn no_final_empty_hunk() -> anyhow::Result<()> { let (testdir, mut ib) = setup(); for i in 0..MAX_ENTRIES_PER_HUNK { ib.push_entry(sample_entry(&format!("/{i:0>10}"))); diff --git a/src/live_tree.rs b/src/live_tree.rs index 67593624..e6697999 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -63,11 +63,11 @@ impl tree::ReadTree for LiveTree { type Entry = EntryValue; type IT = Iter; - fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result { + fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> anyhow::Result { Iter::new(&self.path, subtree, exclude) } - fn estimate_count(&self) -> Result { + fn estimate_count(&self) -> anyhow::Result { // TODO: This stats the file and builds an entry about them, just to // throw it away. We could perhaps change the iter to optionally do // less work. @@ -158,7 +158,7 @@ pub struct Iter { impl Iter { /// Construct a new iter that will visit everything below this root path, /// subject to some exclusions - fn new(root_path: &Path, subtree: Apath, exclude: Exclude) -> Result { + fn new(root_path: &Path, subtree: Apath, exclude: Exclude) -> anyhow::Result { let start_path = subtree.below(root_path); let start_metadata = fs::symlink_metadata(&start_path)?; // Preload iter to return the root and then recurse into it. diff --git a/src/restore.rs b/src/restore.rs index dd480eb5..5b50d19e 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -18,6 +18,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; +use anyhow::{bail, Context}; use filetime::set_file_handle_times; #[cfg(unix)] use filetime::set_symlink_file_times; @@ -65,18 +66,11 @@ pub fn restore( archive: &Archive, destination: &Path, options: &RestoreOptions, -) -> Result { +) -> anyhow::Result { let st = archive.open_stored_tree(options.band_selection.clone())?; - if let Err(source) = ensure_dir_exists(destination) { - return Err(Error::Restore { - path: destination.to_owned(), - source, - }); - } + ensure_dir_exists(destination).context("Creating destination")?; if !options.overwrite && !directory_is_empty(destination)? { - return Err(Error::DestinationNotEmpty { - path: destination.to_owned(), - }); + bail!("Destination directory is not empty") } let mut stats = RestoreStats::default(); let mut bytes_done = 0; diff --git a/src/show.rs b/src/show.rs index cd847a21..fe333be3 100644 --- a/src/show.rs +++ b/src/show.rs @@ -48,7 +48,7 @@ pub fn show_versions( archive: &Archive, options: &ShowVersionsOptions, w: &mut dyn Write, -) -> Result<()> { +) -> anyhow::Result<()> { let mut band_ids = archive.list_band_ids()?; if options.newest_first { band_ids.reverse(); diff --git a/src/stitch.rs b/src/stitch.rs index eaa8d045..514cd7ca 100644 --- a/src/stitch.rs +++ b/src/stitch.rs @@ -174,7 +174,7 @@ mod test { } #[test] - fn stitch_index() -> Result<()> { + fn stitch_index() -> anyhow::Result<()> { // This test uses private interfaces to create an index that breaks // across hunks in a certain way. diff --git a/src/stored_tree.rs b/src/stored_tree.rs index e5cf28fa..73ee0bee 100644 --- a/src/stored_tree.rs +++ b/src/stored_tree.rs @@ -24,6 +24,7 @@ use crate::stored_file::StoredFile; use crate::*; /// Read index and file contents for a version stored in the archive. +#[derive(Debug)] pub struct StoredTree { band: Band, archive: Archive, @@ -31,7 +32,7 @@ pub struct StoredTree { } impl StoredTree { - pub(crate) fn open(archive: &Archive, band_id: &BandId) -> Result { + pub(crate) fn open(archive: &Archive, band_id: &BandId) -> anyhow::Result { Ok(StoredTree { band: Band::open(archive, band_id)?, block_dir: archive.block_dir().clone(), @@ -43,7 +44,7 @@ impl StoredTree { &self.band } - pub fn is_closed(&self) -> Result { + pub fn is_closed(&self) -> anyhow::Result { self.band.is_closed() } @@ -60,14 +61,14 @@ impl ReadTree for StoredTree { /// Return an iter of index entries in this stored tree. // TODO: Should return an iter of Result so that we can inspect them... - fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result { + fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> anyhow::Result { Ok( IterStitchedIndexHunks::new(&self.archive, Some(self.band.id().clone())) .iter_entries(subtree, exclude), ) } - fn estimate_count(&self) -> Result { + fn estimate_count(&self) -> anyhow::Result { self.band.index().estimate_entry_count() } } @@ -112,11 +113,12 @@ mod test { #[test] pub fn cant_open_no_versions() { let af = ScratchArchive::new(); - match af.open_stored_tree(BandSelectionPolicy::Latest) { - Err(Error::ArchiveEmpty) => (), - Err(other) => panic!("unexpected result {other:?}"), - Ok(_) => panic!("unexpected success"), - } + assert_eq!( + af.open_stored_tree(BandSelectionPolicy::Latest) + .unwrap_err() + .to_string(), + "Archive is empty" + ); } #[test] diff --git a/src/transport.rs b/src/transport.rs index 25e27350..24e8fe4b 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -95,22 +95,10 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn read_file(&self, path: &str) -> io::Result; /// Check if a directory exists. - fn is_dir(&self, path: &str) -> io::Result { - match self.metadata(path) { - Ok(metadata) => Ok(metadata.kind == Kind::Dir), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false), - Err(err) => Err(err), - } - } + fn is_dir(&self, path: &str) -> anyhow::Result; /// Check if a regular file exists. - fn is_file(&self, path: &str) -> io::Result { - match self.metadata(path) { - Ok(metadata) => Ok(metadata.kind == Kind::File), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false), - Err(err) => Err(err), - } - } + fn is_file(&self, path: &str) -> anyhow::Result; /// Create a directory, if it does not exist. /// @@ -129,10 +117,10 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn write_file(&self, relpath: &str, content: &[u8]) -> io::Result<()>; /// Get metadata about a file. - fn metadata(&self, relpath: &str) -> io::Result; + fn metadata(&self, relpath: &str) -> anyhow::Result; /// Delete a file. - fn remove_file(&self, relpath: &str) -> io::Result<()>; + fn remove_file(&self, relpath: &str) -> anyhow::Result<()>; /// Delete an empty directory. fn remove_dir(&self, relpath: &str) -> anyhow::Result<()>; diff --git a/src/transport/local.rs b/src/transport/local.rs index c58f6ef0..d627eb3a 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -76,14 +76,16 @@ impl Transport for LocalTransport { Ok(out_buf.into()) } - fn is_file(&self, relpath: &str) -> io::Result { + fn is_file(&self, relpath: &str) -> anyhow::Result { increment_counter!("conserve.local_transport.metadata_reads"); - Ok(self.full_path(relpath).is_file()) + let path = self.full_path(relpath); + Ok(path.is_file()) } - fn is_dir(&self, relpath: &str) -> io::Result { + fn is_dir(&self, relpath: &str) -> anyhow::Result { increment_counter!("conserve.local_transport.metadata_reads"); - Ok(self.full_path(relpath).is_dir()) + let path = self.full_path(relpath); + Ok(path.is_dir()) } fn create_dir(&self, relpath: &str) -> io::Result<()> { @@ -119,8 +121,9 @@ impl Transport for LocalTransport { } } - fn remove_file(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_file(self.full_path(relpath)) + fn remove_file(&self, relpath: &str) -> anyhow::Result<()> { + let path = self.full_path(relpath); + std::fs::remove_file(&path).with_context(|| format!("Delete file {path:?}")) } fn remove_dir(&self, relpath: &str) -> anyhow::Result<()> { @@ -137,9 +140,12 @@ impl Transport for LocalTransport { }) } - fn metadata(&self, relpath: &str) -> io::Result { + fn metadata(&self, relpath: &str) -> anyhow::Result { increment_counter!("conserve.local_transport.metadata_reads"); - let fsmeta = self.root.join(relpath).metadata()?; + let path = self.root.join(relpath); + let fsmeta = path + .metadata() + .with_context(|| format!("Failed to get metadata for {path:?}"))?; Ok(Metadata { len: fsmeta.len(), kind: fsmeta.file_type().into(), diff --git a/src/tree.rs b/src/tree.rs index b35e3422..dcb22217 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -29,16 +29,16 @@ pub trait ReadTree { /// Errors reading individual paths or directories are sent to the UI and /// counted, but are not treated as fatal, and don't appear as Results in the /// iterator. - fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> Result; + fn iter_entries(&self, subtree: Apath, exclude: Exclude) -> anyhow::Result; /// Estimate the number of entries in the tree. /// This might do somewhat expensive IO, so isn't the Iter's `size_hint`. - fn estimate_count(&self) -> Result; + fn estimate_count(&self) -> anyhow::Result; /// Measure the tree size. /// /// This typically requires walking all entries, which may take a while. - fn size(&self, exclude: Exclude) -> Result { + fn size(&self, exclude: Exclude) -> anyhow::Result { let mut files = 0; let mut total_bytes = 0u64; let bar = Bar::new(); diff --git a/src/validate.rs b/src/validate.rs index b1121d7a..7b99dd53 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -36,7 +36,7 @@ pub struct ValidateOptions { pub(crate) fn validate_bands( archive: &Archive, band_ids: &[BandId], -) -> Result> { +) -> anyhow::Result> { let mut block_lens = HashMap::new(); let start = Instant::now(); let total_bands = band_ids.len(); @@ -78,7 +78,7 @@ fn merge_block_lens(into: &mut HashMap, from: &HashMap Result> { +fn validate_stored_tree(st: &StoredTree) -> anyhow::Result> { let mut block_lens = HashMap::new(); // TODO: Check other entry properties are correct. // TODO: Check they're in apath order. diff --git a/tests/api/damaged.rs b/tests/api/damaged.rs index 726d602a..308952a0 100644 --- a/tests/api/damaged.rs +++ b/tests/api/damaged.rs @@ -21,7 +21,7 @@ use conserve::*; #[traced_test] #[test] -fn missing_block_when_checking_hashes() -> Result<()> { +fn missing_block_when_checking_hashes() -> anyhow::Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; archive.validate(&ValidateOptions::default())?; assert!(logs_contain( @@ -31,7 +31,7 @@ fn missing_block_when_checking_hashes() -> Result<()> { #[traced_test] #[test] -fn missing_block_skip_block_hashes() -> Result<()> { +fn missing_block_skip_block_hashes() -> anyhow::Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; archive.validate(&ValidateOptions { skip_block_hashes: true, diff --git a/tests/api/gc.rs b/tests/api/gc.rs index 3e6ab5f8..ea510def 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -99,10 +99,10 @@ fn backup_prevented_by_gc_lock() -> anyhow::Result<()> { // Backup should fail while gc lock is held. let backup_result = backup(&archive, tf.path(), &BackupOptions::default()); - match backup_result { - Err(Error::GarbageCollectionLockHeld) => (), - other => panic!("unexpected result {other:?}"), - }; + assert_eq!( + backup_result.unwrap_err().to_string(), + "Archive is locked for garbage collection" + ); // Leak the lock, then gc breaking the lock. std::mem::forget(lock1); diff --git a/tests/api/restore.rs b/tests/api/restore.rs index 7282742f..34c8769e 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -97,7 +97,10 @@ pub fn decline_to_overwrite() { let restore_err_str = restore(&af, destdir.path(), &options) .expect_err("restore should fail if the destination exists") .to_string(); - assert!(restore_err_str.contains("Destination directory not empty")); + assert!( + restore_err_str.contains("Destination directory is not empty"), + "Unexpected error message: {restore_err_str:?}" + ); } #[test] diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 8ba510dc..079c179d 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -276,7 +276,9 @@ fn basic_backup() { .arg(restore_dir.path()) .assert() .failure() - .stderr(predicate::str::contains("Destination directory not empty")); + .stderr(predicate::str::contains( + "Destination directory is not empty", + )); // Restore with specified band id / backup version. { @@ -317,14 +319,14 @@ fn empty_archive() { .arg(restore_dir.path()) .assert() .failure() - .stderr(predicate::str::contains("Archive has no bands")); + .stderr(predicate::str::contains("Archive is empty")); run_conserve() .arg("ls") .arg(&adir) .assert() .failure() - .stderr(predicate::str::contains("Archive has no bands")); + .stderr(predicate::str::contains("Archive is empty")); run_conserve() .arg("versions") From 8ef96543762a1d05cfebf0a33eba8938d733b50e Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Tue, 22 Aug 2023 08:28:23 -0700 Subject: [PATCH 56/86] WIP add transport::Error to allow understanding lower level erros --- src/blockdir.rs | 46 ++++++++++++++++-------------------- src/errors.rs | 17 -------------- src/stored_file.rs | 4 ++-- src/transport.rs | 53 ++++++++++++++++++++++++++++++++++++++++-- src/transport/local.rs | 12 ++++++---- src/tree.rs | 6 ++--- 6 files changed, 83 insertions(+), 55 deletions(-) diff --git a/src/blockdir.rs b/src/blockdir.rs index 943efdb1..093bcda5 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -30,7 +30,7 @@ use std::sync::Arc; use std::time::Instant; use ::metrics::{counter, histogram, increment_counter}; -use anyhow::Context; +use anyhow::{bail, Context}; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; use rayon::prelude::*; @@ -114,7 +114,11 @@ impl BlockDir { } /// Returns the number of compressed bytes. - pub(crate) fn compress_and_store(&mut self, in_buf: &[u8], hash: &BlockHash) -> Result { + pub(crate) fn compress_and_store( + &mut self, + in_buf: &[u8], + hash: &BlockHash, + ) -> anyhow::Result { // TODO: Move this to a BlockWriter, which can hold a reusable buffer. let mut compressor = Compressor::new(); let uncomp_len = in_buf.len() as u64; @@ -130,16 +134,13 @@ impl BlockDir { histogram!("conserve.block.write_compressed_bytes", comp_len as f64); self.transport .write_file(&relpath, compressed) - .or_else(|io_err| { - if io_err.kind() == io::ErrorKind::AlreadyExists { + .or_else(|err| { + if err.kind() == transport::ErrorKind::AlreadyExists { // Perhaps it was simultaneously created by another thread or process. debug!("Unexpected late detection of existing block {hex_hash:?}"); Ok(()) } else { - Err(Error::WriteBlock { - hash: hex_hash, - source: io_err, - }) + Err(err).with_context(|| format!("Failed to write block {hex_hash:?}")) } })?; Ok(comp_len) @@ -181,16 +182,15 @@ impl BlockDir { /// Read back the contents of a block, as a byte array. /// /// To read a whole file, use StoredFile instead. - pub fn get(&self, address: &Address) -> Result<(Vec, Sizes)> { + pub fn get(&self, address: &Address) -> anyhow::Result<(Vec, Sizes)> { let (mut decompressed, sizes) = self.get_block_content(&address.hash)?; let len = address.len as usize; let start = address.start as usize; let actual_len = decompressed.len(); if (start + len) > actual_len { - return Err(Error::AddressTooLong { - address: address.to_owned(), - actual_len, - }); + bail!("Block {address:?} is only {actual_len} bytes long but address references range {start}..{end}", + start = start, + end = start + len); } if start != 0 { let trimmed = decompressed[start..(start + len)].to_owned(); @@ -316,20 +316,17 @@ impl BlockDir { /// Return the entire contents of the block. /// /// Checks that the hash is correct with the contents. - pub fn get_block_content(&self, hash: &BlockHash) -> Result<(Vec, Sizes)> { + pub fn get_block_content(&self, hash: &BlockHash) -> anyhow::Result<(Vec, Sizes)> { // TODO: Reuse decompressor buffer. // TODO: Reuse read buffer. // TODO: Most importantly, cache decompressed blocks! increment_counter!("conserve.block.read"); let mut decompressor = Decompressor::new(); let block_relpath = block_relpath(hash); - let compressed_bytes = - self.transport - .read_file(&block_relpath) - .map_err(|source| Error::ReadBlock { - source, - hash: hash.to_string(), - })?; + let compressed_bytes = self + .transport + .read_file(&block_relpath) + .with_context(|| format!("Failed to read block {hash}"))?; let decompressed_bytes = decompressor.decompress(&compressed_bytes)?; let actual_hash = BlockHash::from(blake2b::blake2b( BLAKE_HASH_SIZE_BYTES, @@ -337,11 +334,8 @@ impl BlockDir { decompressed_bytes, )); if actual_hash != *hash { - error!("Block file {block_relpath:?} has actual decompressed hash {actual_hash}"); - return Err(Error::BlockCorrupt { - hash: hash.to_string(), - actual_hash: actual_hash.to_string(), - }); + error!(%hash, %actual_hash, %block_relpath, "Block file has wrong hash"); + bail!("Block file {block_relpath:?} has actual decompressed hash {actual_hash}"); } let sizes = Sizes { uncompressed: decompressed_bytes.len() as u64, diff --git a/src/errors.rs b/src/errors.rs index 3dcc58e2..33df04c9 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -19,29 +19,12 @@ use std::path::PathBuf; use thiserror::Error; -use crate::blockdir::Address; use crate::*; /// Conserve specific error. #[non_exhaustive] #[derive(Debug, Error)] pub enum Error { - #[error("Block file {hash:?} corrupt; actual hash {actual_hash:?}")] - BlockCorrupt { hash: String, actual_hash: String }, - - #[error("{address:?} extends beyond decompressed block length {actual_len:?}")] - AddressTooLong { address: Address, actual_len: usize }, - - // TODO: Merge with AddressTooLong - #[error( - "block {block_hash} actual length is {actual_len} but indexes reference {referenced_len}" - )] - ShortBlock { - block_hash: BlockHash, - actual_len: usize, - referenced_len: u64, - }, - #[error("Failed to write block {hash:?}")] WriteBlock { hash: String, source: io::Error }, diff --git a/src/stored_file.rs b/src/stored_file.rs index 506e33c4..35ca69b7 100644 --- a/src/stored_file.rs +++ b/src/stored_file.rs @@ -42,11 +42,11 @@ impl StoredFile { } impl ReadBlocks for StoredFile { - fn num_blocks(&self) -> Result { + fn num_blocks(&self) -> anyhow::Result { Ok(self.addrs.len()) } - fn read_block(&self, i: usize) -> Result<(Vec, Sizes)> { + fn read_block(&self, i: usize) -> anyhow::Result<(Vec, Sizes)> { self.block_dir.get(&self.addrs[i]) } } diff --git a/src/transport.rs b/src/transport.rs index 24e8fe4b..795e9288 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -14,8 +14,8 @@ //! //! Transport operations return std::io::Result to reflect their narrower focus. -use std::io; use std::path::Path; +use std::{fmt, io}; use anyhow::bail; use bytes::Bytes; @@ -114,7 +114,7 @@ pub trait Transport: Send + Sync + std::fmt::Debug { /// then renamed. /// /// If a temporary file is used, the name should start with `crate::TMP_PREFIX`. - fn write_file(&self, relpath: &str, content: &[u8]) -> io::Result<()>; + fn write_file(&self, relpath: &str, content: &[u8]) -> Result<()>; /// Get metadata about a file. fn metadata(&self, relpath: &str) -> anyhow::Result; @@ -163,3 +163,52 @@ pub struct ListDirNames { pub files: Vec, pub dirs: Vec, } + +/// A transport error, as a generalization of IO errors. +#[derive(Debug)] +pub struct Error { + url: Url, + kind: ErrorKind, + source: Option, +} + +impl Error { + pub fn kind(&self) -> ErrorKind { + self.kind + } + + pub(self) fn io_error(path: &Path, source: io::Error) -> Error { + let kind = match source.kind() { + io::ErrorKind::NotFound => ErrorKind::NotFound, + io::ErrorKind::AlreadyExists => ErrorKind::AlreadyExists, + _ => ErrorKind::Other, + }; + Error { + url: Url::from_file_path(path).expect("Convert path to URL"), + kind, + source: Some(source.into()), + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // source is not in the short format; maybe should be in the alternate format? + format!("{kind:?}: {url}", kind = self.kind, url = self.url).fmt(f) + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_ref().map(|e| e.as_ref()) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ErrorKind { + NotFound, + AlreadyExists, + Other, +} + +type Result = std::result::Result; diff --git a/src/transport/local.rs b/src/transport/local.rs index d627eb3a..7e62ab1e 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -98,7 +98,7 @@ impl Transport for LocalTransport { }) } - fn write_file(&self, relpath: &str, content: &[u8]) -> io::Result<()> { + fn write_file(&self, relpath: &str, content: &[u8]) -> super::Result<()> { increment_counter!("conserve.local_transport.write_files"); counter!( "conserve.local_transport.write_file_bytes", @@ -106,16 +106,18 @@ impl Transport for LocalTransport { ); let full_path = self.full_path(relpath); let dir = full_path.parent().unwrap(); + let context = |err| super::Error::io_error(&full_path, err); let mut temp = tempfile::Builder::new() .prefix(crate::TMP_PREFIX) - .tempfile_in(dir)?; + .tempfile_in(dir) + .map_err(context)?; if let Err(err) = temp.write_all(content) { let _ = temp.close(); - return Err(err); + return Err(context(err)); } if let Err(persist_error) = temp.persist(&full_path) { - persist_error.file.close()?; - Err(persist_error.error) + persist_error.file.close().map_err(context)?; + Err(context(persist_error.error)) } else { Ok(()) } diff --git a/src/tree.rs b/src/tree.rs index dcb22217..9e907c6f 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -63,15 +63,15 @@ pub trait ReadTree { /// shouldn't assume the size. pub trait ReadBlocks { /// Return a range of integers indexing the blocks (starting from 0.) - fn num_blocks(&self) -> Result; + fn num_blocks(&self) -> anyhow::Result; - fn block_range(&self) -> Result> { + fn block_range(&self) -> anyhow::Result> { Ok(0..self.num_blocks()?) } /// Read one block and return it as a byte vec. Also returns the compressed and uncompressed /// sizes. - fn read_block(&self, i: usize) -> Result<(Vec, Sizes)>; + fn read_block(&self, i: usize) -> anyhow::Result<(Vec, Sizes)>; } /// The measured size of a tree. From 08d3d4189a421d5275cbf8232be2801fec1b380e Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 24 Aug 2023 19:05:16 -0700 Subject: [PATCH 57/86] Use anyhow more --- src/archive.rs | 36 ++++++++++-------------- src/band.rs | 54 ++++++++++++++++++------------------ src/blockdir.rs | 1 - src/index.rs | 9 ++---- src/jsonio.rs | 58 ++++++++++++++++++++++++++------------- src/transport.rs | 2 +- tests/api/archive.rs | 11 +++----- tests/api/backup.rs | 2 +- tests/api/format_flags.rs | 11 +++----- tests/api/gc.rs | 2 +- tests/cli/main.rs | 4 ++- 11 files changed, 96 insertions(+), 94 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index a7b004c5..ecbb911f 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -14,14 +14,13 @@ //! Archives holding backup material. use std::collections::{HashMap, HashSet}; -use std::io::ErrorKind; use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; -use anyhow::anyhow; +use anyhow::{anyhow, bail, ensure, Context}; use itertools::Itertools; use rayon::prelude::*; use serde::{Deserialize, Serialize}; @@ -61,18 +60,18 @@ pub struct DeleteOptions { impl Archive { /// Make a new archive in a local directory. - pub fn create_path(path: &Path) -> Result { + pub fn create_path(path: &Path) -> anyhow::Result { Archive::create(Box::new(LocalTransport::new(path))) } /// Make a new archive in a new directory accessed by a Transport. - pub fn create(transport: Box) -> Result { + pub fn create(transport: Box) -> anyhow::Result { transport .create_dir("") - .map_err(|source| Error::CreateArchiveDirectory { source })?; + .context("Failed to create archive directory")?; let names = transport.list_dir_names("").map_err(Error::from)?; if !names.files.is_empty() || !names.dirs.is_empty() { - return Err(Error::NewArchiveDirectoryNotEmpty); + bail!("New archive directory is not empty"); } let block_dir = BlockDir::create(transport.sub_transport(BLOCK_DIR))?; write_json( @@ -91,25 +90,18 @@ impl Archive { /// Open an existing archive. /// /// Checks that the header is correct. - pub fn open_path(path: &Path) -> Result { + pub fn open_path(path: &Path) -> anyhow::Result { Archive::open(Box::new(LocalTransport::new(path))) } - pub fn open(transport: Box) -> Result { - let header: ArchiveHeader = - read_json(&transport, HEADER_FILENAME).map_err(|err| match err { - Error::MetadataNotFound { .. } => Error::NotAnArchive {}, - Error::IOError { source } if source.kind() == ErrorKind::NotFound => { - Error::NotAnArchive {} - } - Error::IOError { source } => Error::ReadArchiveHeader { source }, - other => other, - })?; - if header.conserve_archive_version != ARCHIVE_VERSION { - return Err(Error::UnsupportedArchiveVersion { - version: header.conserve_archive_version, - }); - } + pub fn open(transport: Box) -> anyhow::Result { + let header: ArchiveHeader = read_json(&transport, HEADER_FILENAME) + .context("Failed to read archive header (maybe this is not a Conserve archive?)")?; + ensure!( + header.conserve_archive_version == ARCHIVE_VERSION, + "Unsupported archive version {:?}", + header.conserve_archive_version + ); let block_dir = BlockDir::open(transport.sub_transport(BLOCK_DIR)); Ok(Archive { block_dir, diff --git a/src/band.rs b/src/band.rs index a71e1f45..55be0e24 100644 --- a/src/band.rs +++ b/src/band.rs @@ -23,7 +23,7 @@ use std::borrow::Cow; -use anyhow::Context; +use anyhow::{ensure, Context}; use itertools::Itertools; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -128,14 +128,14 @@ impl Band { /// Make a new band (and its on-disk directory). /// /// The Band gets the next id after those that already exist. - pub fn create(archive: &Archive) -> Result { + pub fn create(archive: &Archive) -> anyhow::Result { Band::create_with_flags(archive, flags::DEFAULT) } pub fn create_with_flags( archive: &Archive, format_flags: &[Cow<'static, str>], - ) -> Result { + ) -> anyhow::Result { format_flags .iter() .for_each(|f| assert!(flags::SUPPORTED.contains(&f.as_ref()), "unknown flag {f:?}")); @@ -146,7 +146,7 @@ impl Band { transport .create_dir("") .and_then(|()| transport.create_dir(INDEX_DIR)) - .map_err(|source| Error::CreateBand { source })?; + .context("Failed to create band directory")?; let band_format_version = if format_flags.is_empty() { Some("0.6.3".to_owned()) } else { @@ -157,7 +157,7 @@ impl Band { band_format_version, format_flags: format_flags.into(), }; - write_json(&transport, BAND_HEAD_FILENAME, &head)?; + write_json(&transport, BAND_HEAD_FILENAME, &head).context("Failed to write band header")?; Ok(Band { band_id, head, @@ -166,7 +166,7 @@ impl Band { } /// Mark this band closed: no more blocks should be written after this. - pub fn close(&self, index_hunk_count: u64) -> Result<()> { + pub fn close(&self, index_hunk_count: u64) -> anyhow::Result<()> { write_json( &self.transport, BAND_TAIL_FILENAME, @@ -175,19 +175,18 @@ impl Band { index_hunk_count: Some(index_hunk_count), }, ) + .context("Failed to write band tail") } /// Open the band with the given id. - pub fn open(archive: &Archive, band_id: &BandId) -> Result { + pub fn open(archive: &Archive, band_id: &BandId) -> anyhow::Result { let transport: Box = archive.transport().sub_transport(&band_id.to_string()); let head: Head = read_json(&transport, BAND_HEAD_FILENAME)?; if let Some(version) = &head.band_format_version { - if !band_version_supported(version) { - return Err(Error::UnsupportedBandVersion { - band_id: band_id.to_owned(), - version: version.to_owned(), - }); - } + ensure!( + band_version_supported(version), + "Unsupported band version {version:?} in {band_id}" + ); } else { debug!("Old(?) band {band_id} has no format version"); // Unmarked, old bands, are accepted for now. In the next archive @@ -200,13 +199,10 @@ impl Band { .filter(|f| !flags::SUPPORTED.contains(&f.as_ref())) .cloned() .collect_vec(); - if !unsupported_flags.is_empty() { - return Err(Error::UnsupportedBandFormatFlags { - band_id: band_id.clone(), - unsupported_flags, - }); - } - + ensure!( + unsupported_flags.is_empty(), + "Unsupported band format flags {unsupported_flags:?} in {band_id}" + ); Ok(Band { band_id: band_id.to_owned(), head, @@ -251,17 +247,19 @@ impl Band { } /// Return info about the state of this band. - pub fn get_info(&self) -> Result { + pub fn get_info(&self) -> anyhow::Result { let tail_option: Option = match read_json(&self.transport, BAND_TAIL_FILENAME) { Ok(tail) => Some(tail), - Err(Error::MetadataNotFound { .. }) => None, - Err(err) => return Err(err), + Err(jsonio::Error::NotFound { .. }) => None, + Err(err) => return Err(err.into()), }; let start_time = OffsetDateTime::from_unix_timestamp(self.head.start_time) - .expect("invalid band start timestamp"); - let end_time = tail_option.as_ref().map(|tail| { - OffsetDateTime::from_unix_timestamp(tail.end_time).expect("invalid end timestamp") - }); + .context("invalid band start timestamp {self.head.start_time:?}")?; + let end_time = tail_option + .as_ref() + .map(|tail| OffsetDateTime::from_unix_timestamp(tail.end_time)) + .transpose() + .context("invalid end timestamp {tail.end_time:?}")?; Ok(Info { id: self.band_id.clone(), is_closed: tail_option.is_some(), @@ -361,7 +359,7 @@ mod tests { let e = Band::open(&af, &BandId::zero()); let e_str = e.unwrap_err().to_string(); assert!( - e_str.contains("Band version \"8888.8.8\" in"), + e_str.contains("Unsupported band version \"8888.8.8\" in b0000"), "bad band version: {e_str:#?}" ); } diff --git a/src/blockdir.rs b/src/blockdir.rs index 093bcda5..8fdfd34a 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -23,7 +23,6 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryInto; -use std::io; use std::path::Path; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; diff --git a/src/index.rs b/src/index.rs index a710d9af..103b30b6 100644 --- a/src/index.rs +++ b/src/index.rs @@ -20,6 +20,7 @@ use std::path::Path; use std::sync::Arc; use std::vec; +use anyhow::Context; use metrics::{counter, increment_counter}; use time::OffsetDateTime; use tracing::error; @@ -251,21 +252,17 @@ impl IndexWriter { self.check_order.check(&self.entries.last().unwrap().apath); } let relpath = hunk_relpath(self.sequence); - let write_error = |source| Error::WriteIndex { - path: relpath.clone(), - source, - }; let json = serde_json::to_vec(&self.entries).map_err(|source| Error::SerializeIndex { source })?; if (self.sequence % HUNKS_PER_SUBDIR) == 0 { self.transport .create_dir(&subdir_relpath(self.sequence)) - .map_err(write_error)?; + .context("Failed to create index subdir")?; } let compressed_bytes = self.compressor.compress(&json)?; self.transport .write_file(&relpath, compressed_bytes) - .map_err(write_error)?; + .context("Failed to write index hunk")?; self.stats.index_hunks += 1; self.stats.compressed_index_bytes += compressed_bytes.len() as u64; diff --git a/src/jsonio.rs b/src/jsonio.rs index dd19c3e1..61d419ad 100644 --- a/src/jsonio.rs +++ b/src/jsonio.rs @@ -1,5 +1,5 @@ // Conserve backup system. -// Copyright 2015, 2016, 2018, 2020 Martin Pool. +// Copyright 2015, 2016, 2018, 2020, 2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -14,12 +14,41 @@ //! Read and write JSON files. use std::io; +use std::path::PathBuf; use serde::de::DeserializeOwned; -use crate::errors::Error; -use crate::transport::Transport; -use crate::Result; +use crate::transport::{self, Transport}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("File not found: {path:?}")] + NotFound { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error("IO error")] + Io { + #[from] + source: io::Error, + }, + + #[error("JSON serialization error")] + Json { + #[from] + source: serde_json::Error, + }, + + #[error("Transport error")] + Transport { + #[from] + source: transport::Error, + }, +} + +pub type Result = std::result::Result; /// Write uncompressed json to a file on a Transport. pub(crate) fn write_json(transport: &TR, relpath: &str, obj: &T) -> Result<()> @@ -27,18 +56,12 @@ where T: serde::Serialize, TR: AsRef, { - let mut s: String = serde_json::to_string(&obj).map_err(|source| Error::SerializeJson { - path: relpath.to_string(), - source, - })?; + let mut s: String = serde_json::to_string(&obj)?; s.push('\n'); transport .as_ref() .write_file(relpath, s.as_bytes()) - .map_err(|source| Error::WriteMetadata { - path: relpath.to_owned(), - source, - }) + .map_err(Error::from) } /// Read and deserialize uncompressed json from a Transport. @@ -51,16 +74,13 @@ where .as_ref() .read_file(path) .map_err(|err| match err.kind() { - io::ErrorKind::NotFound => Error::MetadataNotFound { - path: path.to_owned(), + io::ErrorKind::NotFound => Error::NotFound { + path: PathBuf::from(path), source: err, }, - _ => err.into(), + _ => Error::from(err), })?; - serde_json::from_slice(&bytes).map_err(|source| Error::DeserializeJson { - source, - path: path.into(), - }) + serde_json::from_slice(&bytes).map_err(Error::from) } #[cfg(test)] diff --git a/src/transport.rs b/src/transport.rs index 795e9288..770133ea 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -204,7 +204,7 @@ impl std::error::Error for Error { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum ErrorKind { NotFound, AlreadyExists, diff --git a/tests/api/archive.rs b/tests/api/archive.rs index d186fa9b..5742453c 100644 --- a/tests/api/archive.rs +++ b/tests/api/archive.rs @@ -23,7 +23,6 @@ use conserve::archive::Archive; use conserve::test_fixtures::ScratchArchive; use conserve::Band; use conserve::BandId; -use conserve::Error; #[test] fn create_then_open_archive() { @@ -46,12 +45,10 @@ fn fails_on_non_empty_directory() { temp.child("i am already here").touch().unwrap(); let result = Archive::create_path(temp.path()); - assert!(result.is_err()); - if let Err(Error::NewArchiveDirectoryNotEmpty) = result { - } else { - panic!("expected an error for a non-empty new archive directory") - } - + assert_eq!( + result.unwrap_err().to_string(), + "New archive directory is not empty" + ); temp.close().unwrap(); } diff --git a/tests/api/backup.rs b/tests/api/backup.rs index a8f01249..2124a0d3 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -55,7 +55,7 @@ pub fn simple_backup() { #[test] #[traced_test] -pub fn simple_backup_with_excludes() -> Result<()> { +pub fn simple_backup_with_excludes() -> anyhow::Result<()> { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_file("hello"); diff --git a/tests/api/format_flags.rs b/tests/api/format_flags.rs index 9a4b1160..0649f4d0 100644 --- a/tests/api/format_flags.rs +++ b/tests/api/format_flags.rs @@ -3,8 +3,6 @@ //! Tests for per-band format flags. -use assert_matches::assert_matches; - use conserve::test_fixtures::ScratchArchive; use conserve::*; @@ -49,9 +47,8 @@ fn unknown_format_flag_fails_to_open() { .unwrap(); let err = Band::open(&af, &BandId::zero()).unwrap_err(); - println!("{err}"); - assert_matches!(err, Error::UnsupportedBandFormatFlags { .. }); - assert!(err - .to_string() - .starts_with(r#"Band b0000 has feature flags ["wibble"] not supported by Conserve "#)); + assert_eq!( + err.to_string(), + "Unsupported band format flags [\"wibble\"] in b0000" + ) } diff --git a/tests/api/gc.rs b/tests/api/gc.rs index ea510def..2114bde6 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -1,4 +1,4 @@ -// Copyright 2015, 2016, 2017, 2019, 2020, 2021 Martin Pool. +// Copyright 2015-2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 079c179d..c3ba8c7e 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -81,7 +81,9 @@ fn clean_error_on_non_archive() { .arg(".") .assert() .failure() - .stderr(predicate::str::contains("Not a Conserve archive")); + .stderr(predicate::str::contains( + "Failed to read archive header (maybe this is not a Conserve archive?)", + )); } #[test] From be4af0ca46935b964806b827cd0f9249e98ccb1f Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Thu, 24 Aug 2023 19:13:36 -0700 Subject: [PATCH 58/86] More conversion to anyhow --- src/archive.rs | 19 ++++++++++--------- src/backup.rs | 22 +++++++++------------- src/blockdir.rs | 6 +++--- src/errors.rs | 37 ------------------------------------- 4 files changed, 22 insertions(+), 62 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index ecbb911f..67c27fe6 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -27,7 +27,6 @@ use serde::{Deserialize, Serialize}; use tracing::{debug, error, warn}; use crate::blockhash::BlockHash; -use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; use crate::progress::{Bar, Progress}; @@ -69,7 +68,9 @@ impl Archive { transport .create_dir("") .context("Failed to create archive directory")?; - let names = transport.list_dir_names("").map_err(Error::from)?; + let names = transport + .list_dir_names("") + .context("Failed to list new archive directory")?; if !names.files.is_empty() || !names.dirs.is_empty() { bail!("New archive directory is not empty"); } @@ -135,7 +136,7 @@ impl Archive { } /// Returns a vector of band ids, in sorted order from first to last. - pub fn list_band_ids(&self) -> Result> { + pub fn list_band_ids(&self) -> anyhow::Result> { let mut band_ids: Vec = self.iter_band_ids_unsorted()?.collect(); band_ids.sort_unstable(); Ok(band_ids) @@ -166,13 +167,13 @@ impl Archive { /// Return an iterator of valid band ids in this archive, in arbitrary order. /// /// Errors reading the archive directory are logged and discarded. - fn iter_band_ids_unsorted(&self) -> Result> { + fn iter_band_ids_unsorted(&self) -> anyhow::Result> { // This doesn't check for extraneous files or directories, which should be a weird rare // problem. Validate does. Ok(self .transport .list_dir_names("") - .map_err(|source| Error::ListBands { source })? + .context("Failed to list archive directory")? .dirs .into_iter() .filter(|dir_name| dir_name != BLOCK_DIR) @@ -181,7 +182,7 @@ impl Archive { /// Return the `BandId` of the highest-numbered band, or Ok(None) if there /// are no bands, or an Err if any occurred reading the directory. - pub fn last_band_id(&self) -> Result> { + pub fn last_band_id(&self) -> anyhow::Result> { Ok(self.iter_band_ids_unsorted()?.max()) } @@ -228,7 +229,7 @@ impl Archive { } /// Returns an iterator of blocks that are present and referenced by no index. - pub fn unreferenced_blocks(&self) -> Result> { + pub fn unreferenced_blocks(&self) -> anyhow::Result> { let referenced = self.referenced_blocks(&self.list_band_ids()?)?; Ok(self .block_dir() @@ -375,14 +376,14 @@ impl Archive { Ok(()) } - fn validate_archive_dir(&self) -> Result<()> { + fn validate_archive_dir(&self) -> anyhow::Result<()> { // TODO: More tests for the problems detected here. debug!("Check archive directory..."); let mut seen_bands = HashSet::::new(); for entry_result in self .transport .iter_dir_entries("") - .map_err(|source| Error::ListBands { source })? + .context("Failed to list archive directory")? { match entry_result { Ok(DirEntry { diff --git a/src/backup.rs b/src/backup.rs index e57392ce..3fc1f7b7 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -20,7 +20,7 @@ use std::io::prelude::*; use std::path::Path; use std::time::{Duration, Instant}; -use anyhow::bail; +use anyhow::{bail, Context}; use derive_more::{Add, AddAssign}; use itertools::Itertools; use tracing::error; @@ -293,12 +293,8 @@ fn store_file_content( let mut buffer = Vec::new(); let mut addresses = Vec::
::with_capacity(1); loop { - read_with_retries(&mut buffer, MAX_BLOCK_SIZE, from_file).map_err(|source| { - Error::StoreFile { - apath: apath.to_owned(), - source, - } - })?; + read_with_retries(&mut buffer, MAX_BLOCK_SIZE, from_file) + .with_context(|| format!("Failed to read contents of source file {apath:?}"))?; if buffer.is_empty() { break; } @@ -413,12 +409,12 @@ impl FileCombiner { return Ok(()); } self.buf.resize(start + expected_len, 0); - let len = from_file - .read(&mut self.buf[start..]) - .map_err(|source| Error::StoreFile { - apath: entry.apath().to_owned(), - source, - })?; + let len = from_file.read(&mut self.buf[start..]).with_context(|| { + format!( + "Failed to read contents of source file {apath:?}", + apath = entry.apath, + ) + })?; self.buf.truncate(start + len); if len == 0 { self.stats.empty_files += 1; diff --git a/src/blockdir.rs b/src/blockdir.rs index 8fdfd34a..bc9f1cef 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -99,14 +99,14 @@ impl BlockDir { } /// Create a BlockDir directory and return an object accessing it. - pub fn create_path(path: &Path) -> Result { + pub fn create_path(path: &Path) -> anyhow::Result { BlockDir::create(Box::new(LocalTransport::new(path))) } - pub fn create(transport: Box) -> Result { + pub fn create(transport: Box) -> anyhow::Result { transport .create_dir("") - .map_err(|source| Error::CreateBlockDir { source })?; + .context("Failed to create blockdir")?; Ok(BlockDir { transport: Arc::from(transport), }) diff --git a/src/errors.rs b/src/errors.rs index 33df04c9..73ff20d8 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -25,37 +25,6 @@ use crate::*; #[non_exhaustive] #[derive(Debug, Error)] pub enum Error { - #[error("Failed to write block {hash:?}")] - WriteBlock { hash: String, source: io::Error }, - - #[error("Failed to read block {hash:?}")] - ReadBlock { hash: String, source: io::Error }, - - #[error("Block {block_hash} is missing")] - BlockMissing { block_hash: BlockHash }, - - #[error("Failed to list block files")] - ListBlocks { source: io::Error }, - - #[error("Not a Conserve archive")] - NotAnArchive {}, - - #[error("Failed to read archive header")] - ReadArchiveHeader { source: io::Error }, - - #[error( - "Archive version {:?} is not supported by Conserve {}", - version, - crate::version() - )] - UnsupportedArchiveVersion { version: String }, - - #[error( - "Band version {version:?} in {band_id} is not supported by Conserve {}", - crate::version() - )] - UnsupportedBandVersion { band_id: BandId, version: String }, - #[error( "Band {band_id} has feature flags {unsupported_flags:?} \ not supported by Conserve {conserve_version}", @@ -69,9 +38,6 @@ pub enum Error { #[error("Destination directory not empty: {:?}", path)] DestinationNotEmpty { path: PathBuf }, - #[error("Archive has no bands")] - ArchiveEmpty, - #[error("Directory for new archive is not empty")] NewArchiveDirectoryNotEmpty, @@ -147,9 +113,6 @@ pub enum Error { #[error("Metadata file not found: {:?}", path)] MetadataNotFound { path: String, source: io::Error }, - #[error("Failed to list bands")] - ListBands { source: io::Error }, - #[error("Failed to read source file {:?}", path)] ReadSourceFile { path: PathBuf, source: io::Error }, From 58fca86c6446e0ce8e1beabe461b133e49a590f6 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 07:30:31 -0700 Subject: [PATCH 59/86] WIP: Declare a transport::Error --- src/transport.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/src/transport.rs b/src/transport.rs index 1f6bb745..d4b1d9b7 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -14,8 +14,8 @@ //! //! Transport operations return std::io::Result to reflect their narrower focus. -use std::io; use std::path::Path; +use std::{error, fmt, io}; use bytes::Bytes; use url::Url; @@ -28,7 +28,7 @@ use local::LocalTransport; /// Open a `Transport` to access a local directory. /// /// `s` may be a local path or a URL. -pub fn open_transport(s: &str) -> Result> { +pub fn open_transport(s: &str) -> crate::Result> { if let Ok(url) = Url::parse(s) { match url.scheme() { "file" => Ok(Box::new(LocalTransport::new( @@ -38,7 +38,7 @@ pub fn open_transport(s: &str) -> Result> { // Probably a Windows path with drive letter, like "c:/thing", not actually a URL. Ok(Box::new(LocalTransport::new(Path::new(s)))) } - other => Err(Error::UrlScheme { + other => Err(crate::Error::UrlScheme { scheme: other.to_owned(), }), } @@ -176,3 +176,54 @@ pub struct ListDirNames { pub files: Vec, pub dirs: Vec, } + +/// A transport error, as a generalization of IO errors. +#[derive(Debug)] +pub struct Error { + pub url: Url, + pub kind: ErrorKind, + /// Might be for example an IO error or S3 error. + pub source: Option>, +} + +impl Error { + pub fn kind(&self) -> ErrorKind { + self.kind + } + + pub(self) fn io_error(path: &Path, source: io::Error) -> Error { + let kind = match source.kind() { + io::ErrorKind::NotFound => ErrorKind::NotFound, + io::ErrorKind::AlreadyExists => ErrorKind::AlreadyExists, + _ => ErrorKind::Other, + }; + Error { + url: Url::from_file_path(path).expect("Convert path to URL"), + kind, + source: Some(source.into()), + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // source is not in the short format; maybe should be in the alternate format? + format!("{kind:?}: {url}", kind = self.kind, url = self.url).fmt(f) + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_ref().map(|e| e.as_ref()) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum ErrorKind { + // TODO: Manual Display, or maybe use a macro crate? + NotFound, + AlreadyExists, + Other, +} + +type Result = std::result::Result; From bcc3cf0302491e0dd2eba4720e10261a8428c047 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 07:42:10 -0700 Subject: [PATCH 60/86] cargo update --- Cargo.lock | 650 +++++++++++++++++++++++------------------------------ 1 file changed, 278 insertions(+), 372 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7d5a288..47b649a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,48 +15,38 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "aho-corasick" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -67,17 +57,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -91,9 +81,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle", "bstr", @@ -163,6 +153,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "blake2-rfc" version = "0.2.18" @@ -175,13 +171,12 @@ dependencies = [ [[package]] name = "bstr" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", - "once_cell", - "regex-automata", + "regex-automata 0.3.6", "serde", ] @@ -214,9 +209,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -226,9 +224,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" +checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" dependencies = [ "clap_builder", "clap_derive", @@ -237,13 +235,12 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" +checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" dependencies = [ "anstream", "anstyle", - "bitflags", "clap_lex", "strsim", "terminal_size", @@ -251,21 +248,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" dependencies = [ "heck", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "clicolors-control" @@ -380,34 +377,33 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.8.0", + "memoffset 0.9.0", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] [[package]] -name = "ctor" -version = "0.1.26" +name = "deranged" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" dependencies = [ - "quote 1.0.27", - "syn 1.0.109", + "serde", ] [[package]] @@ -417,8 +413,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.66", + "quote 1.0.33", "rustc_version", "syn 1.0.109", ] @@ -449,9 +445,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "endian-type" @@ -461,13 +457,13 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -482,23 +478,20 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", - "windows-sys 0.48.0", + "redox_syscall", + "windows-sys", ] [[package]] @@ -518,18 +511,18 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -538,11 +531,11 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "bstr", "fnv", "log", @@ -555,7 +548,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "ignore", "walkdir", ] @@ -568,9 +561,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ "ahash", ] @@ -592,18 +585,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -613,9 +597,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -650,40 +634,19 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2cb48b81b1dc9f39676bf99f5499babfec7cd8fe14307f7b3d747208fb5690" - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] +checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi 0.3.2", "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "is-terminal" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" -dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -697,15 +660,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -718,9 +681,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" @@ -734,11 +697,17 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" + [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -746,12 +715,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "mach2" @@ -768,7 +734,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -788,18 +754,18 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "metrics" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ "ahash", "metrics-macros", @@ -812,21 +778,21 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "metrics-util" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.2", + "hashbrown 0.13.1", "indexmap", "metrics", "num_cpus", @@ -857,7 +823,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.7.1", @@ -889,9 +855,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", "libm", @@ -899,11 +865,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.2", "libc", ] @@ -918,8 +884,8 @@ dependencies = [ [[package]] name = "nutmeg" -version = "0.1.3-pre" -source = "git+https://github.com/sourcefrog/nutmeg#2d10f07580e038040f9ea72b0cc9cc65b3316104" +version = "0.1.3" +source = "git+https://github.com/sourcefrog/nutmeg#ebee48a9b964271e0668d60aa57f05190aba3cce" dependencies = [ "atty", "parking_lot", @@ -929,28 +895,19 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-float" -version = "3.7.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc2dbde8f8a79f2102cc474ceb0ad68e3b80b85289ea62389b60e66777e4213" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "num-traits", ] -[[package]] -name = "output_vt100" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" -dependencies = [ - "winapi", -] - [[package]] name = "overload" version = "0.1.1" @@ -969,28 +926,28 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1000,9 +957,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "portable-atomic" -version = "1.3.2" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59d1bcc64fc5d021d67521f818db868368028108d37f0e98d74e33f68297b5" +checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" [[package]] name = "ppv-lite86" @@ -1043,13 +1000,11 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" dependencies = [ - "ctor", "diff", - "output_vt100", "yansi", ] @@ -1064,9 +1019,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -1078,7 +1033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" dependencies = [ "bit-set", - "bitflags", + "bitflags 1.3.2", "byteorder", "lazy_static", "num-traits", @@ -1104,9 +1059,9 @@ dependencies = [ [[package]] name = "quanta" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc73c42f9314c4bdce450c77e6f09ecbddefbeddb1b5979ded332a3913ded33" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" dependencies = [ "crossbeam-utils", "libc", @@ -1135,11 +1090,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.58", + "proc-macro2 1.0.66", ] [[package]] @@ -1197,7 +1152,7 @@ version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -1228,33 +1183,25 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ea134c32fe12df286020949d57d052a90c4001f2dbec4c1c074f39bcb7fc8c" -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ - "aho-corasick 1.0.1", + "aho-corasick", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -1266,6 +1213,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.4", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -1274,9 +1232,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rstest" @@ -1295,8 +1253,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290ca1a1c8ca7edb7c3283bd44dc35dd54fdec6253a3912e201ba1072018fca8" dependencies = [ "cfg-if", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.66", + "quote 1.0.33", "rustc_version", "syn 1.0.109", "unicode-ident", @@ -1313,16 +1271,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.48.0", + "linux-raw-sys 0.3.8", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "0.38.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys 0.4.5", + "windows-sys", ] [[package]] @@ -1339,9 +1310,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -1354,41 +1325,41 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "9f5db24220c009de9bd45e69fb2938f4b6d2df856aa9304ce377b3180f83b7c1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "5ad697f7e0b65af4983a4ce8f56ed5b357e8d3c36651bf6a7e13639c17b8e670" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -1412,9 +1383,9 @@ checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "snap" @@ -1451,33 +1422,33 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.66", + "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.16" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.66", + "quote 1.0.33", "unicode-ident", ] [[package]] name = "tempfile" -version = "3.5.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", + "redox_syscall", + "rustix 0.38.8", + "windows-sys", ] [[package]] @@ -1486,8 +1457,8 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" dependencies = [ - "rustix", - "windows-sys 0.48.0", + "rustix 0.37.23", + "windows-sys", ] [[package]] @@ -1498,22 +1469,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1534,10 +1505,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.21" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" dependencies = [ + "deranged", "itoa", "libc", "num_threads", @@ -1554,9 +1526,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" dependencies = [ "time-core", ] @@ -1601,13 +1573,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1682,7 +1654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" dependencies = [ "lazy_static", - "quote 1.0.27", + "quote 1.0.33", "syn 1.0.109", ] @@ -1700,9 +1672,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -1721,15 +1693,15 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unix_mode" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35abed4630bb800f02451a7428205d1f37b8e125001471bfab259beee6a587ed" +checksum = "b55eedc365f81a3c32aea49baf23fa965e3cd85bcc28fb8045708c7388d124ef" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -1791,9 +1763,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1801,53 +1773,53 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.27", + "quote 1.0.33", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -1884,137 +1856,71 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "yansi" From 30553b99542b26d217ac2afca57983bd669d3231 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 07:44:20 -0700 Subject: [PATCH 61/86] Define a transport::Error --- src/errors.rs | 6 ++++++ src/transport.rs | 38 +++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 3dcc58e2..b2ebc06e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -219,4 +219,10 @@ pub enum Error { #[from] source: snap::Error, }, + + #[error(transparent)] + Transport { + #[from] + source: transport::Error, + }, } diff --git a/src/transport.rs b/src/transport.rs index d4b1d9b7..ed75d354 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -183,7 +183,21 @@ pub struct Error { pub url: Url, pub kind: ErrorKind, /// Might be for example an IO error or S3 error. - pub source: Option>, + pub source: Option, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum ErrorKind { + // TODO: Manual Display, or maybe use a macro crate? + NotFound, + AlreadyExists, + Other, +} + +#[derive(Debug)] +pub enum ErrorSource { + Io(io::Error), + // S3(s3::Error), } impl Error { @@ -200,7 +214,7 @@ impl Error { Error { url: Url::from_file_path(path).expect("Convert path to URL"), kind, - source: Some(source.into()), + source: Some(ErrorSource::Io(source)), } } } @@ -214,16 +228,18 @@ impl fmt::Display for Error { impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.source.as_ref().map(|e| e.as_ref()) + match self.source { + Some(ErrorSource::Io(ref e)) => Some(e), + // Some(ErrorSource::S3(ref e)) => Some(e), + None => None, + } } } -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ErrorKind { - // TODO: Manual Display, or maybe use a macro crate? - NotFound, - AlreadyExists, - Other, -} - type Result = std::result::Result; + +// impl From for crate::Error { +// fn from(e: Error) -> Self { +// crate::Error:source:Transport { source: e } +// } +// } From fb56bef89efa5ed9a0c070652f299c0da7c4f2b1 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 07:56:55 -0700 Subject: [PATCH 62/86] transport::create_dir -> transport::Result Better display of transport::ErrorKind Add transport::ErrorKind::PermissionDenied --- src/archive.rs | 4 +--- src/band.rs | 3 +-- src/blockdir.rs | 4 +--- src/index.rs | 4 +--- src/transport.rs | 31 ++++++++++++++++--------------- src/transport/local.rs | 18 +++++++++--------- 6 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 961159b9..50920ccf 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -66,9 +66,7 @@ impl Archive { /// Make a new archive in a new directory accessed by a Transport. pub fn create(transport: Box) -> Result { - transport - .create_dir("") - .map_err(|source| Error::CreateArchiveDirectory { source })?; + transport.create_dir("")?; let names = transport.list_dir_names("").map_err(Error::from)?; if !names.files.is_empty() || !names.dirs.is_empty() { return Err(Error::NewArchiveDirectoryNotEmpty); diff --git a/src/band.rs b/src/band.rs index 90525575..c5752f1f 100644 --- a/src/band.rs +++ b/src/band.rs @@ -144,8 +144,7 @@ impl Band { let transport: Box = archive.transport().sub_transport(&band_id.to_string()); transport .create_dir("") - .and_then(|()| transport.create_dir(INDEX_DIR)) - .map_err(|source| Error::CreateBand { source })?; + .and_then(|()| transport.create_dir(INDEX_DIR))?; let band_format_version = if format_flags.is_empty() { Some("0.6.3".to_owned()) } else { diff --git a/src/blockdir.rs b/src/blockdir.rs index e7806f91..90a2db37 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -104,9 +104,7 @@ impl BlockDir { } pub fn create(transport: Box) -> Result { - transport - .create_dir("") - .map_err(|source| Error::CreateBlockDir { source })?; + transport.create_dir("")?; Ok(BlockDir { transport: Arc::from(transport), }) diff --git a/src/index.rs b/src/index.rs index a3b21af7..34c1cb2e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -258,9 +258,7 @@ impl IndexWriter { let json = serde_json::to_vec(&self.entries).map_err(|source| Error::SerializeIndex { source })?; if (self.sequence % HUNKS_PER_SUBDIR) == 0 { - self.transport - .create_dir(&subdir_relpath(self.sequence)) - .map_err(write_error)?; + self.transport.create_dir(&subdir_relpath(self.sequence))?; } let compressed_bytes = self.compressor.compress(&json)?; self.transport diff --git a/src/transport.rs b/src/transport.rs index ed75d354..a4b81862 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -1,4 +1,4 @@ -// Copyright 2020, 2021, 2022 Martin Pool. +// Copyright 2020, 2021, 2022, 2023 Martin Pool. // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -15,9 +15,10 @@ //! Transport operations return std::io::Result to reflect their narrower focus. use std::path::Path; -use std::{error, fmt, io}; +use std::{error, fmt, io, result}; use bytes::Bytes; +use derive_more::Display; use url::Url; use crate::*; @@ -118,7 +119,7 @@ pub trait Transport: Send + Sync + std::fmt::Debug { /// If the directory already exists, it's not an error. /// /// This function does not create missing parent directories. - fn create_dir(&self, relpath: &str) -> io::Result<()>; + fn create_dir(&self, relpath: &str) -> Result<()>; /// Write a complete file. /// @@ -186,11 +187,16 @@ pub struct Error { pub source: Option, } -#[derive(Debug, PartialEq, Eq, Clone, Copy)] +/// General categories of transport errors. +#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)] pub enum ErrorKind { - // TODO: Manual Display, or maybe use a macro crate? + #[display(fmt = "Not found")] NotFound, + #[display(fmt = "Already exists")] AlreadyExists, + #[display(fmt = "Permission denied")] + PermissionDenied, + #[display(fmt = "Other transport error")] Other, } @@ -209,6 +215,7 @@ impl Error { let kind = match source.kind() { io::ErrorKind::NotFound => ErrorKind::NotFound, io::ErrorKind::AlreadyExists => ErrorKind::AlreadyExists, + io::ErrorKind::PermissionDenied => ErrorKind::PermissionDenied, _ => ErrorKind::Other, }; Error { @@ -222,12 +229,12 @@ impl Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // source is not in the short format; maybe should be in the alternate format? - format!("{kind:?}: {url}", kind = self.kind, url = self.url).fmt(f) + format!("{kind}: {url}", kind = self.kind, url = self.url).fmt(f) } } -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { match self.source { Some(ErrorSource::Io(ref e)) => Some(e), // Some(ErrorSource::S3(ref e)) => Some(e), @@ -236,10 +243,4 @@ impl std::error::Error for Error { } } -type Result = std::result::Result; - -// impl From for crate::Error { -// fn from(e: Error) -> Self { -// crate::Error:source:Transport { source: e } -// } -// } +type Result = result::Result; diff --git a/src/transport/local.rs b/src/transport/local.rs index 3df800f5..c79dc8df 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -85,12 +85,13 @@ impl Transport for LocalTransport { Ok(self.full_path(relpath).is_dir()) } - fn create_dir(&self, relpath: &str) -> io::Result<()> { - create_dir(self.full_path(relpath)).or_else(|err| { + fn create_dir(&self, relpath: &str) -> super::Result<()> { + let path = self.full_path(relpath); + create_dir(&path).or_else(|err| { if err.kind() == io::ErrorKind::AlreadyExists { Ok(()) } else { - Err(err) + Err(super::Error::io_error(&path, err)) } }) } @@ -305,15 +306,14 @@ mod test { } #[test] - fn remove_dir_all() -> std::io::Result<()> { + fn remove_dir_all() { let temp = assert_fs::TempDir::new().unwrap(); let transport = LocalTransport::new(temp.path()); - transport.create_dir("aaa")?; - transport.create_dir("aaa/bbb")?; - transport.create_dir("aaa/bbb/ccc")?; + transport.create_dir("aaa").unwrap(); + transport.create_dir("aaa/bbb").unwrap(); + transport.create_dir("aaa/bbb/ccc").unwrap(); - transport.remove_dir_all("aaa")?; - Ok(()) + transport.remove_dir_all("aaa").unwrap(); } } From 7711d169d587537fdb6494b7d798cf674425b55d Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 13:48:14 -0700 Subject: [PATCH 63/86] todo --- src/bandid.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bandid.rs b/src/bandid.rs index 5d9b9e27..eedebcec 100644 --- a/src/bandid.rs +++ b/src/bandid.rs @@ -21,6 +21,7 @@ use serde::Serialize; use crate::errors::Error; /// Identifier for a band within an archive, eg 'b0001'. +// TODO: `impl Copy`, because it's just an int. #[derive(Debug, PartialEq, Clone, Eq, PartialOrd, Ord, Hash, Serialize)] pub struct BandId(u32); From 314e515c86f4389f8114dbbae6d3bf08beadea18 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 13:49:08 -0700 Subject: [PATCH 64/86] transport::Error from remove_dir, remove_dir_all Specific Error::BandNotFound --- src/band.rs | 8 +++++--- src/errors.rs | 3 +++ src/transport.rs | 4 ++-- src/transport/local.rs | 11 +++++++---- tests/cli/delete.rs | 6 +----- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/band.rs b/src/band.rs index c5752f1f..da035758 100644 --- a/src/band.rs +++ b/src/band.rs @@ -218,9 +218,11 @@ impl Band { archive .transport() .remove_dir_all(&band_id.to_string()) - .map_err(|source| Error::BandDeletion { - band_id: band_id.clone(), - source, + .map_err(|err| match err.kind { + transport::ErrorKind::NotFound => Error::BandNotFound { + band_id: band_id.clone(), + }, + _ => Error::from(err), }) } diff --git a/src/errors.rs b/src/errors.rs index b2ebc06e..cc194db3 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -164,6 +164,9 @@ pub enum Error { #[error("Metadata file not found: {:?}", path)] MetadataNotFound { path: String, source: io::Error }, + #[error("Band not found: {band_id}")] + BandNotFound { band_id: BandId }, + #[error("Failed to list bands")] ListBands { source: io::Error }, diff --git a/src/transport.rs b/src/transport.rs index a4b81862..f604869b 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -137,10 +137,10 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn remove_file(&self, relpath: &str) -> io::Result<()>; /// Delete an empty directory. - fn remove_dir(&self, relpath: &str) -> io::Result<()>; + fn remove_dir(&self, relpath: &str) -> self::Result<()>; /// Delete a directory and all its contents. - fn remove_dir_all(&self, relpath: &str) -> io::Result<()>; + fn remove_dir_all(&self, relpath: &str) -> self::Result<()>; /// Make a new transport addressing a subdirectory. fn sub_transport(&self, relpath: &str) -> Box; diff --git a/src/transport/local.rs b/src/transport/local.rs index c79dc8df..2cf6682b 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -123,12 +123,14 @@ impl Transport for LocalTransport { std::fs::remove_file(self.full_path(relpath)) } - fn remove_dir(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_dir(self.full_path(relpath)) + fn remove_dir(&self, relpath: &str) -> super::Result<()> { + let path = self.full_path(relpath); + std::fs::remove_dir(&path).map_err(|err| super::Error::io_error(&path, err)) } - fn remove_dir_all(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_dir_all(self.full_path(relpath)) + fn remove_dir_all(&self, relpath: &str) -> super::Result<()> { + let path = self.full_path(relpath); + std::fs::remove_dir_all(&path).map_err(|err| super::Error::io_error(&path, err)) } fn sub_transport(&self, relpath: &str) -> Box { @@ -151,6 +153,7 @@ impl Transport for LocalTransport { } fn url(&self) -> String { + // TODO: An actual URL. self.root.to_string_lossy().into() } } diff --git a/tests/cli/delete.rs b/tests/cli/delete.rs index ab21574b..b3791e5e 100644 --- a/tests/cli/delete.rs +++ b/tests/cli/delete.rs @@ -124,11 +124,7 @@ fn delete_nonexistent_band() { .arg(af.path()) .assert() .stderr(predicate::str::contains( - "ERROR conserve: Failed to delete band b0000", + "ERROR conserve: Band not found: b0000", )) - .stderr( - predicate::str::is_match(r"caused by: (File not found.|No such file or directory|The system cannot find the file specified\.) \(os error \d+\)") - .unwrap(), - ) .failure(); } From d857a2336d72121a477be2d5e45d7c6b5999c6b8 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 13:56:13 -0700 Subject: [PATCH 65/86] More transport::Error in metadata etc --- src/errors.rs | 3 --- src/index.rs | 6 +----- src/transport.rs | 12 ++++++------ src/transport/local.rs | 16 +++++++++------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index cc194db3..f528a8f4 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -191,9 +191,6 @@ pub enum Error { #[error("Failed to restore modification time on {:?}", path)] RestoreModificationTime { path: PathBuf, source: io::Error }, - #[error("Failed to delete band {}", band_id)] - BandDeletion { band_id: BandId, source: io::Error }, - #[error("Unsupported URL scheme {:?}", scheme)] UrlScheme { scheme: String }, diff --git a/src/index.rs b/src/index.rs index 34c1cb2e..499e794f 100644 --- a/src/index.rs +++ b/src/index.rs @@ -310,11 +310,7 @@ impl IndexRead { // one hunk being missing. for i in 0.. { let path = hunk_relpath(i); - if !self - .transport - .is_file(&path) - .map_err(|source| Error::ReadIndex { source, path })? - { + if !self.transport.is_file(&path)? { // If hunk 1 is missing, 1 hunks exists. return Ok(i); } diff --git a/src/transport.rs b/src/transport.rs index f604869b..17d31fb2 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -97,19 +97,19 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn read_file(&self, path: &str) -> io::Result; /// Check if a directory exists. - fn is_dir(&self, path: &str) -> io::Result { + fn is_dir(&self, path: &str) -> Result { match self.metadata(path) { Ok(metadata) => Ok(metadata.kind == Kind::Dir), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), Err(err) => Err(err), } } /// Check if a regular file exists. - fn is_file(&self, path: &str) -> io::Result { + fn is_file(&self, path: &str) -> Result { match self.metadata(path) { Ok(metadata) => Ok(metadata.kind == Kind::File), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), Err(err) => Err(err), } } @@ -131,10 +131,10 @@ pub trait Transport: Send + Sync + std::fmt::Debug { fn write_file(&self, relpath: &str, content: &[u8]) -> io::Result<()>; /// Get metadata about a file. - fn metadata(&self, relpath: &str) -> io::Result; + fn metadata(&self, relpath: &str) -> Result; /// Delete a file. - fn remove_file(&self, relpath: &str) -> io::Result<()>; + fn remove_file(&self, relpath: &str) -> self::Result<()>; /// Delete an empty directory. fn remove_dir(&self, relpath: &str) -> self::Result<()>; diff --git a/src/transport/local.rs b/src/transport/local.rs index 2cf6682b..271612ec 100644 --- a/src/transport/local.rs +++ b/src/transport/local.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use metrics::{counter, increment_counter}; -use crate::transport::{DirEntry, Metadata, Transport}; +use super::{DirEntry, Error, Metadata, Result, Transport}; #[derive(Clone, Debug)] pub struct LocalTransport { @@ -75,12 +75,12 @@ impl Transport for LocalTransport { Ok(out_buf.into()) } - fn is_file(&self, relpath: &str) -> io::Result { + fn is_file(&self, relpath: &str) -> Result { increment_counter!("conserve.local_transport.metadata_reads"); Ok(self.full_path(relpath).is_file()) } - fn is_dir(&self, relpath: &str) -> io::Result { + fn is_dir(&self, relpath: &str) -> Result { increment_counter!("conserve.local_transport.metadata_reads"); Ok(self.full_path(relpath).is_dir()) } @@ -119,8 +119,9 @@ impl Transport for LocalTransport { } } - fn remove_file(&self, relpath: &str) -> io::Result<()> { - std::fs::remove_file(self.full_path(relpath)) + fn remove_file(&self, relpath: &str) -> super::Result<()> { + let path = self.full_path(relpath); + std::fs::remove_file(&path).map_err(|err| super::Error::io_error(&path, err)) } fn remove_dir(&self, relpath: &str) -> super::Result<()> { @@ -139,9 +140,10 @@ impl Transport for LocalTransport { }) } - fn metadata(&self, relpath: &str) -> io::Result { + fn metadata(&self, relpath: &str) -> Result { increment_counter!("conserve.local_transport.metadata_reads"); - let fsmeta = self.root.join(relpath).metadata()?; + let path = self.root.join(relpath); + let fsmeta = path.metadata().map_err(|err| Error::io_error(&path, err))?; Ok(Metadata { len: fsmeta.len(), kind: fsmeta.file_type().into(), From 2e8c497dd5f738178e0f999b5718210fcc4a1267 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Fri, 25 Aug 2023 17:48:37 -0700 Subject: [PATCH 66/86] More transport errors read_json returns REsult