From a50c6b6aa6492ae1c7e8c080c62d4aa6bcb9ed42 Mon Sep 17 00:00:00 2001 From: quambene Date: Sat, 13 Jan 2024 17:22:16 +0100 Subject: [PATCH 1/7] Clean up todo --- src/cache.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cache.rs b/src/cache.rs index 5fc7e5b..eb8d3c3 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -76,7 +76,6 @@ pub trait Caching { fn exists(&self, bookmark: &TargetBookmark) -> bool; /// Open the cached file for a bookmark. - // TODO: return `Result, anyhow::Error>` (see ). fn open(&self, bookmark: &TargetBookmark) -> Result, BogrepError>; /// Get the content of a bookmark from cache. From 4f4521e43c419cdd710f359d05608a6a0fda0feb Mon Sep 17 00:00:00 2001 From: quambene Date: Sat, 13 Jan 2024 18:11:51 +0100 Subject: [PATCH 2/7] Refactor fetch urls --- benches/fetch.rs | 8 +-- src/bookmarks/mod.rs | 4 +- src/bookmarks/target_bookmarks.rs | 25 ++++++-- src/cmd/fetch.rs | 102 ++++++++++++------------------ src/cmd/import.rs | 3 +- src/cmd/init.rs | 2 +- src/cmd/update.rs | 8 +-- 7 files changed, 73 insertions(+), 79 deletions(-) diff --git a/benches/fetch.rs b/benches/fetch.rs index e1a4380..c3931e2 100644 --- a/benches/fetch.rs +++ b/benches/fetch.rs @@ -1,6 +1,6 @@ use bogrep::{ - cmd, errors::BogrepError, html, Cache, CacheMode, Caching, Fetch, MockClient, TargetBookmark, - TargetBookmarks, + cmd, errors::BogrepError, html, Action, Cache, CacheMode, Caching, Fetch, MockClient, + TargetBookmark, TargetBookmarks, }; use chrono::Utc; use criterion::{criterion_group, criterion_main, Criterion}; @@ -68,7 +68,7 @@ async fn fetch_concurrently(max_concurrent_requests: usize) { None, HashSet::new(), HashSet::new(), - bogrep::Action::Fetch, + bogrep::Action::FetchAndReplace, ), ); } @@ -110,7 +110,7 @@ async fn fetch_in_parallel(max_parallel_requests: usize) { None, HashSet::new(), HashSet::new(), - bogrep::Action::Fetch, + Action::FetchAndReplace, ), ); } diff --git a/src/bookmarks/mod.rs b/src/bookmarks/mod.rs index 568c764..9b56531 100644 --- a/src/bookmarks/mod.rs +++ b/src/bookmarks/mod.rs @@ -18,9 +18,9 @@ use uuid::Uuid; pub enum Action { /// Fetch and cache the bookmark, even if it is cached already. The cached /// content will be updated with the most recent version of the website. - Fetch, + FetchAndReplace, /// Fetch and cache bookmark if it is not cached yet. - Add, + FetchAndAdd, /// Remove bookmark from cache. Remove, /// No actions to be performed. diff --git a/src/bookmarks/target_bookmarks.rs b/src/bookmarks/target_bookmarks.rs index 75089fc..bec2948 100644 --- a/src/bookmarks/target_bookmarks.rs +++ b/src/bookmarks/target_bookmarks.rs @@ -40,6 +40,10 @@ impl TargetBookmark { action, } } + + pub fn set_action(&mut self, action: Action) { + self.action = action; + } } impl From for TargetBookmark { @@ -141,8 +145,13 @@ impl TargetBookmarks { let url = entry.key().clone(); let target_bookmark = entry.into_mut(); debug!("Overwrite duplicate target bookmark: {}", url); - // TODO: use existing bookmark id and bookmark urls - *target_bookmark = bookmark; + + // We are keeping the existing id and url, but overwriting all other fields. + target_bookmark.last_imported = bookmark.last_imported; + target_bookmark.last_cached = bookmark.last_cached; + target_bookmark.sources = bookmark.sources; + target_bookmark.cache_modes = bookmark.cache_modes; + target_bookmark.action = bookmark.action; } Entry::Vacant(entry) => { entry.insert(bookmark); @@ -214,7 +223,7 @@ impl TargetBookmarks { None, bookmark.sources.to_owned(), HashSet::new(), - Action::Add, + Action::FetchAndAdd, ); self.insert(target_bookmark); } @@ -368,9 +377,15 @@ mod tests { let res = target_bookmarks.update(&source_bookmarks); assert!(res.is_ok()); assert_eq!(target_bookmarks.get(url1).unwrap().action, Action::None); - assert_eq!(target_bookmarks.get(url2).unwrap().action, Action::Add); + assert_eq!( + target_bookmarks.get(url2).unwrap().action, + Action::FetchAndAdd + ); assert_eq!(target_bookmarks.get(url3).unwrap().action, Action::None); - assert_eq!(target_bookmarks.get(url4).unwrap().action, Action::Add); + assert_eq!( + target_bookmarks.get(url4).unwrap().action, + Action::FetchAndAdd + ); assert_eq!(target_bookmarks.get(url5).unwrap().action, Action::Remove); } diff --git a/src/cmd/fetch.rs b/src/cmd/fetch.rs index 47a2f24..074b611 100644 --- a/src/cmd/fetch.rs +++ b/src/cmd/fetch.rs @@ -22,26 +22,16 @@ pub async fn fetch(config: &Config, args: &FetchArgs) -> Result<(), anyhow::Erro let mut target_reader = utils::open_file_in_read_mode(&config.target_bookmark_file)?; let mut target_writer = utils::open_and_truncate_file(&config.target_bookmark_lock_file)?; - if args.urls.is_empty() { - fetch_bookmarks( - &client, - &cache, - &mut target_reader, - &mut target_writer, - config.settings.max_concurrent_requests, - args.all, - ) - .await?; - } else { - fetch_urls( - &args.urls, - &client, - &cache, - &mut target_reader, - &mut target_writer, - ) - .await?; - } + fetch_bookmarks( + &client, + &cache, + &mut target_reader, + &mut target_writer, + config.settings.max_concurrent_requests, + args.all, + &args.urls, + ) + .await?; utils::close_and_rename( (target_writer, &config.target_bookmark_lock_file), @@ -51,36 +41,6 @@ pub async fn fetch(config: &Config, args: &FetchArgs) -> Result<(), anyhow::Erro Ok(()) } -pub async fn fetch_urls( - urls: &[String], - client: &impl Fetch, - cache: &impl Caching, - target_reader: &mut impl ReadTarget, - target_writer: &mut impl WriteTarget, -) -> Result<(), anyhow::Error> { - let now = Utc::now(); - let mut target_bookmarks = TargetBookmarks::default(); - target_reader.read(&mut target_bookmarks)?; - - for url in urls { - let mut bookmark = TargetBookmark::new( - url, - now, - None, - HashSet::new(), - HashSet::new(), - Action::Fetch, - ); - fetch_and_cache_bookmark(client, cache, &mut bookmark).await?; - println!("Fetched website for {url}"); - target_bookmarks.insert(bookmark); - } - - target_writer.write(&target_bookmarks)?; - - Ok(()) -} - pub async fn fetch_bookmarks( client: &impl Fetch, cache: &impl Caching, @@ -88,7 +48,9 @@ pub async fn fetch_bookmarks( target_writer: &mut impl WriteTarget, max_concurrent_requests: usize, fetch_all: bool, + urls: &[String], ) -> Result<(), anyhow::Error> { + let now = Utc::now(); let mut target_bookmarks = TargetBookmarks::default(); target_reader.read(&mut target_bookmarks)?; @@ -98,9 +60,25 @@ pub async fn fetch_bookmarks( } if fetch_all { - target_bookmarks.set_action(&Action::Fetch); + target_bookmarks.set_action(&Action::FetchAndReplace); } else { - target_bookmarks.set_action(&Action::Add); + target_bookmarks.set_action(&Action::FetchAndAdd); + } + + for url in urls { + if let Some(target_bookmark) = target_bookmarks.get_mut(url) { + target_bookmark.set_action(Action::FetchAndReplace); + } else { + let target_bookmark = TargetBookmark::new( + url, + now, + None, + HashSet::new(), + HashSet::new(), + Action::FetchAndReplace, + ); + target_bookmarks.insert(target_bookmark); + } } fetch_and_cache_bookmarks( @@ -215,13 +193,13 @@ async fn fetch_and_cache_bookmark( bookmark: &mut TargetBookmark, ) -> Result<(), BogrepError> { match bookmark.action { - Action::Fetch => { + Action::FetchAndReplace => { let website = client.fetch(bookmark).await?; trace!("Fetched website: {website}"); let html = html::filter_html(&website)?; cache.replace(html, bookmark).await?; } - Action::Add => { + Action::FetchAndAdd => { if !cache.exists(bookmark) { let website = client.fetch(bookmark).await?; trace!("Fetched website: {website}"); @@ -310,7 +288,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Fetch, + action: Action::FetchAndReplace, }, ), ( @@ -322,7 +300,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Fetch, + action: Action::FetchAndReplace, }, ), ])); @@ -374,7 +352,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Fetch, + action: Action::FetchAndReplace, }, ), ( @@ -386,7 +364,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Fetch, + action: Action::FetchAndReplace, }, ), ])); @@ -438,7 +416,7 @@ mod tests { last_cached: Some(now), sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Add, + action: Action::FetchAndAdd, }, ), ( @@ -450,7 +428,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Add, + action: Action::FetchAndAdd, }, ), ])); @@ -512,7 +490,7 @@ mod tests { last_cached: Some(now), sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Add, + action: Action::FetchAndAdd, }, ), ( @@ -524,7 +502,7 @@ mod tests { last_cached: None, sources: HashSet::new(), cache_modes: HashSet::new(), - action: Action::Add, + action: Action::FetchAndAdd, }, ), ])); diff --git a/src/cmd/import.rs b/src/cmd/import.rs index 02e2a84..1053af9 100644 --- a/src/cmd/import.rs +++ b/src/cmd/import.rs @@ -63,7 +63,8 @@ fn log_import(source_reader: &[SourceReader], target_bookmarks: &TargetBookmarks "Imported {} bookmarks from {} {source}: {}", target_bookmarks .values() - .filter(|bookmark| bookmark.action == Action::Fetch || bookmark.action == Action::Add) + .filter(|bookmark| bookmark.action == Action::FetchAndReplace + || bookmark.action == Action::FetchAndAdd) .collect::>() .len(), source_reader.len(), diff --git a/src/cmd/init.rs b/src/cmd/init.rs index 2a311c1..669db9f 100644 --- a/src/cmd/init.rs +++ b/src/cmd/init.rs @@ -62,7 +62,7 @@ async fn init_bookmarks( let mut target_bookmarks = TargetBookmarks::from(source_bookmarks); - target_bookmarks.set_action(&Action::Add); + target_bookmarks.set_action(&Action::FetchAndAdd); println!( "Imported {} bookmarks from {} sources: {}", diff --git a/src/cmd/update.rs b/src/cmd/update.rs index ad435bd..f77875d 100644 --- a/src/cmd/update.rs +++ b/src/cmd/update.rs @@ -108,7 +108,7 @@ mod tests { last_cached: Some(now.timestamp_millis()), sources: HashSet::new(), cache_modes: HashSet::from_iter([CacheMode::Html]), - action: Action::Add, + action: Action::FetchAndAdd, }),("https://www.quantamagazine.org/how-mathematical-curves-power-cryptography-20220919/".to_owned(), TargetBookmark { id: "25b6357e-6eda-4367-8212-84376c6efe05".to_owned(), @@ -117,7 +117,7 @@ mod tests { last_cached: Some(now.timestamp_millis()), sources: HashSet::new(), cache_modes: HashSet::from_iter([CacheMode::Html]), - action: Action::Add, + action: Action::FetchAndAdd, }), ]), ); @@ -232,7 +232,7 @@ mod tests { last_cached: Some(now.timestamp_millis()), sources: HashSet::new(), cache_modes: HashSet::from_iter([CacheMode::Text]), - action: Action::Add, + action: Action::FetchAndAdd, }), ("https://www.quantamagazine.org/how-mathematical-curves-power-cryptography-20220919/".to_owned(), TargetBookmark { id: "25b6357e-6eda-4367-8212-84376c6efe05".to_owned(), url: "https://www.quantamagazine.org/how-mathematical-curves-power-cryptography-20220919/".to_owned(), @@ -240,7 +240,7 @@ mod tests { last_cached: Some(now.timestamp_millis()), sources: HashSet::new(), cache_modes: HashSet::from_iter([CacheMode::Text]), - action: Action::Add, + action: Action::FetchAndAdd, })])); for url in &expected_bookmarks { client From 30127a66dc280398056ced7a1ed961aaf0b94f24 Mon Sep 17 00:00:00 2001 From: quambene Date: Sat, 13 Jan 2024 18:12:45 +0100 Subject: [PATCH 3/7] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6a7b0..752bf46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Add benchmarks for fetching - changed - Update to rust 1.75 + - Fix duplicate cache files for `bogrep fetch --urls` ### v0.6.1 From b77dfa8e8a1f924074ec84f9d77fd345a1b2b833 Mon Sep 17 00:00:00 2001 From: quambene Date: Sat, 13 Jan 2024 18:32:48 +0100 Subject: [PATCH 4/7] Take ignored urls into account --- CHANGELOG.md | 1 + src/bookmarks/target_bookmarks.rs | 57 +++++++++++++++++++++++++++++++ src/cmd/import.rs | 28 ++++++++++++--- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 752bf46..02fa7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - added - Add `action` to `TargetBookmark` - Add benchmarks for fetching + - Take ignored urls into account in `bogrep import` - changed - Update to rust 1.75 - Fix duplicate cache files for `bogrep fetch --urls` diff --git a/src/bookmarks/target_bookmarks.rs b/src/bookmarks/target_bookmarks.rs index bec2948..aab4b1c 100644 --- a/src/bookmarks/target_bookmarks.rs +++ b/src/bookmarks/target_bookmarks.rs @@ -181,6 +181,14 @@ impl TargetBookmarks { } } + pub fn ignore_urls(&mut self, ignored_urls: &[String]) { + for url in ignored_urls { + if let Some(target_bookmark) = self.get_mut(url) { + target_bookmark.set_action(Action::Remove) + } + } + } + pub fn filter_to_add<'a>( &self, source_bookmarks: &'a SourceBookmarks, @@ -389,6 +397,55 @@ mod tests { assert_eq!(target_bookmarks.get(url5).unwrap().action, Action::Remove); } + #[test] + fn test_ignore_urls() { + let now = Utc::now(); + let url1 = "https://url1.com"; + let url2 = "https://url2.com"; + let url3 = "https://url3.com"; + let ignored_urls = vec![url1.to_owned(), url3.to_owned()]; + let mut target_bookmarks = TargetBookmarks::new(HashMap::from_iter([ + ( + url1.to_owned(), + TargetBookmark::new( + url1, + now, + None, + HashSet::new(), + HashSet::new(), + Action::None, + ), + ), + ( + url2.to_owned(), + TargetBookmark::new( + url2, + now, + None, + HashSet::new(), + HashSet::new(), + Action::None, + ), + ), + ( + url3.to_owned(), + TargetBookmark::new( + url3, + now, + None, + HashSet::new(), + HashSet::new(), + Action::None, + ), + ), + ])); + + target_bookmarks.ignore_urls(&ignored_urls); + assert!(target_bookmarks.get(url1).unwrap().action == Action::Remove); + assert!(target_bookmarks.get(url2).unwrap().action == Action::None); + assert!(target_bookmarks.get(url3).unwrap().action == Action::Remove); + } + #[test] fn test_read_target_bookmarks() { let expected_bookmarks = EXPECTED_BOOKMARKS.as_bytes().to_vec(); diff --git a/src/cmd/import.rs b/src/cmd/import.rs index 1053af9..264eb4f 100644 --- a/src/cmd/import.rs +++ b/src/cmd/import.rs @@ -19,7 +19,12 @@ pub fn import(config: &Config, args: ImportArgs) -> Result<(), anyhow::Error> { let mut target_reader = utils::open_file_in_read_mode(&config.target_bookmark_file)?; let mut target_writer = utils::open_and_truncate_file(&config.target_bookmark_lock_file)?; - import_source(source_reader, &mut target_reader, &mut target_writer)?; + import_source( + source_reader, + &mut target_reader, + &mut target_writer, + &config.settings.ignored_urls, + )?; utils::close_and_rename( (target_writer, &config.target_bookmark_lock_file), @@ -33,6 +38,7 @@ fn import_source( mut source_reader: Vec, target_reader: &mut impl ReadTarget, target_writer: &mut impl WriteTarget, + ignored_urls: &[String], ) -> Result<(), anyhow::Error> { let mut source_bookmarks = SourceBookmarks::default(); @@ -41,10 +47,12 @@ fn import_source( } let mut target_bookmarks = TargetBookmarks::default(); - target_reader.read(&mut target_bookmarks)?; + target_bookmarks.update(&source_bookmarks)?; + target_bookmarks.ignore_urls(ignored_urls); target_bookmarks.clean_up(); + target_writer.write(&target_bookmarks)?; log_import(&source_reader, &target_bookmarks); @@ -99,9 +107,15 @@ mod tests { // Set cursor position to the start again to prepare cursor for reading. target_reader.set_position(0); let mut target_writer = Cursor::new(Vec::new()); + let ignored_urls = vec![]; let source_reader = SourceReader::init(source).unwrap(); - let res = import_source(vec![source_reader], &mut target_reader, &mut target_writer); + let res = import_source( + vec![source_reader], + &mut target_reader, + &mut target_writer, + &ignored_urls, + ); assert!(res.is_ok(), "{}", res.unwrap_err()); let actual = target_writer.into_inner(); @@ -136,8 +150,14 @@ mod tests { Box::new(source_reader_writer.clone()), Box::new(bookmark_reader), ); + let ignored_urls = vec![]; - let res = import_source(vec![source_reader], target_reader, target_writer); + let res = import_source( + vec![source_reader], + target_reader, + target_writer, + &ignored_urls, + ); let actual = target_writer.get_ref(); let actual_bookmarks = json::deserialize::(actual); From 546608248f84957b19ddba29dbe0d466545d327e Mon Sep 17 00:00:00 2001 From: quambene Date: Sat, 13 Jan 2024 19:10:08 +0100 Subject: [PATCH 5/7] Set source to internal --- src/bookmarks/target_bookmarks.rs | 4 ++++ src/cmd/fetch.rs | 38 +++++++++++++++++-------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/bookmarks/target_bookmarks.rs b/src/bookmarks/target_bookmarks.rs index aab4b1c..3ffe979 100644 --- a/src/bookmarks/target_bookmarks.rs +++ b/src/bookmarks/target_bookmarks.rs @@ -44,6 +44,10 @@ impl TargetBookmark { pub fn set_action(&mut self, action: Action) { self.action = action; } + + pub fn set_source(&mut self, source: SourceType) { + self.sources.insert(source); + } } impl From for TargetBookmark { diff --git a/src/cmd/fetch.rs b/src/cmd/fetch.rs index 074b611..ec76e66 100644 --- a/src/cmd/fetch.rs +++ b/src/cmd/fetch.rs @@ -3,7 +3,8 @@ use crate::{ bookmarks::Action, cache::CacheMode, errors::BogrepError, - html, utils, Cache, Caching, Client, Config, Fetch, FetchArgs, TargetBookmark, TargetBookmarks, + html, utils, Cache, Caching, Client, Config, Fetch, FetchArgs, SourceType, TargetBookmark, + TargetBookmarks, }; use chrono::Utc; use colored::Colorize; @@ -61,26 +62,29 @@ pub async fn fetch_bookmarks( if fetch_all { target_bookmarks.set_action(&Action::FetchAndReplace); + } else if !urls.is_empty() { + for url in urls { + if let Some(target_bookmark) = target_bookmarks.get_mut(url) { + target_bookmark.set_action(Action::FetchAndReplace); + target_bookmark.set_source(SourceType::Internal); + } else { + let mut sources = HashSet::new(); + sources.insert(SourceType::Internal); + let target_bookmark = TargetBookmark::new( + url, + now, + None, + sources, + HashSet::new(), + Action::FetchAndReplace, + ); + target_bookmarks.insert(target_bookmark); + } + } } else { target_bookmarks.set_action(&Action::FetchAndAdd); } - for url in urls { - if let Some(target_bookmark) = target_bookmarks.get_mut(url) { - target_bookmark.set_action(Action::FetchAndReplace); - } else { - let target_bookmark = TargetBookmark::new( - url, - now, - None, - HashSet::new(), - HashSet::new(), - Action::FetchAndReplace, - ); - target_bookmarks.insert(target_bookmark); - } - } - fetch_and_cache_bookmarks( client, cache, From 726a3ae1d756cdce57dd47838619877bbb124791 Mon Sep 17 00:00:00 2001 From: quambene Date: Sun, 14 Jan 2024 11:56:55 +0100 Subject: [PATCH 6/7] Remove integration test feature --- .github/workflows/rust-ci.yml | 16 ++++++++-------- CHANGELOG.md | 2 ++ Cargo.toml | 3 --- README.md | 10 +++++----- tests/test_add.rs | 1 - tests/test_clean.rs | 2 -- tests/test_config.rs | 1 - tests/test_configure.rs | 5 ----- tests/test_fetch.rs | 3 --- tests/test_import.rs | 7 ------- tests/test_init.rs | 1 - tests/test_remove.rs | 1 - tests/test_search.rs | 3 --- tests/test_update.rs | 1 - 14 files changed, 15 insertions(+), 41 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 60d7888..68b7fe3 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -48,15 +48,15 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - name: cargo test - run: cargo test --locked + - name: cargo test --lib + run: cargo test --lib --locked integration-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - name: cargo test --test '*' --features integration-test - run: cargo test --locked --test '*' --features integration-test + - name: cargo test --test '*' + run: cargo test --test '*' --locked os-test: runs-on: ${{ matrix.os }} name: os-test / ${{ matrix.os }} @@ -67,10 +67,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - name: cargo test - run: cargo test --locked - - name: cargo test --test '*' --features integration-test - run: cargo test --locked --test '*' --features integration-test + - name: cargo test --lib + run: cargo test --lib --locked + - name: cargo test --test '*' + run: cargo test --test '*' --locked doc-test: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 02fa7b6..b49f762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - changed - Update to rust 1.75 - Fix duplicate cache files for `bogrep fetch --urls` +- removed + - Remove `integration-test` feature ### v0.6.1 diff --git a/Cargo.toml b/Cargo.toml index 046d7f4..7ed0ba7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,9 +12,6 @@ categories = ["command-line-utilities", "text-processing"] readme = "README.md" license = "Apache-2.0" -[features] -integration-test = [] - [[bench]] name = "fetch" harness = false diff --git a/README.md b/README.md index 0d1cf76..43dfdd6 100644 --- a/README.md +++ b/README.md @@ -250,12 +250,12 @@ You can configure the configuration path via the environment variable ## Testing ``` bash -# Run unit tests +# Run unit tests and integration tests cargo test -# Run integration tests -cargo test --test '*' --features integration-test +# Run unit tests +cargo test --lib -# Run unit and integration tests -cargo test --features integration-test +# Run integration tests +cargo test --test '*' ``` diff --git a/tests/test_add.rs b/tests/test_add.rs index 3a9f362..09bde44 100644 --- a/tests/test_add.rs +++ b/tests/test_add.rs @@ -4,7 +4,6 @@ use predicates::str; use tempfile::tempdir; #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_add() { let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); diff --git a/tests/test_clean.rs b/tests/test_clean.rs index 2f267e0..11553f3 100644 --- a/tests/test_clean.rs +++ b/tests/test_clean.rs @@ -9,7 +9,6 @@ use std::{ use tempfile::tempdir; #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_clean() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; @@ -79,7 +78,6 @@ async fn test_clean() { } #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_clean_all() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; diff --git a/tests/test_config.rs b/tests/test_config.rs index 85d8bf6..3bcffb0 100644 --- a/tests/test_config.rs +++ b/tests/test_config.rs @@ -3,7 +3,6 @@ use std::env; use tempfile::tempdir; #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_config() { let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); diff --git a/tests/test_configure.rs b/tests/test_configure.rs index cfbb332..de74f73 100644 --- a/tests/test_configure.rs +++ b/tests/test_configure.rs @@ -51,35 +51,30 @@ fn test_configure_source(source: &str) { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_configure_source_simple() { let source = "./test_data/bookmarks_simple.txt"; test_configure_source(source); } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_configure_source_firefox() { let source = "./test_data/bookmarks_firefox.json"; test_configure_source(source); } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_configure_source_chrome() { let source = "./test_data/bookmarks_chromium.json"; test_configure_source(source); } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_configure_source_chrome_no_extension() { let source = "./test_data/bookmarks_chromium_no_extension"; test_configure_source(source); } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_configure_ignored_urls() { let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); diff --git a/tests/test_fetch.rs b/tests/test_fetch.rs index faaa54f..418f111 100644 --- a/tests/test_fetch.rs +++ b/tests/test_fetch.rs @@ -9,7 +9,6 @@ use std::{ use tempfile::tempdir; #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_fetch() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; @@ -69,7 +68,6 @@ async fn test_fetch() { } #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_fetch_diff() { let mock_server = common::start_mock_server().await; let mock_website_1 = common::mount_mock_scoped(&mock_server, 1, 10).await; @@ -131,7 +129,6 @@ async fn test_fetch_diff() { // Test fetching if the cache directory was removed manually, leading to // inconsistent state as the target bookmarks are still marked as last cached. #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_fetch_empty_cache() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; diff --git a/tests/test_import.rs b/tests/test_import.rs index 45d2461..4426fc6 100644 --- a/tests/test_import.rs +++ b/tests/test_import.rs @@ -39,7 +39,6 @@ fn test_import(source: &str, temp_path: &Path) { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_simple() { let source = "./test_data/bookmarks_simple.txt"; let temp_dir = tempdir().unwrap(); @@ -50,7 +49,6 @@ fn test_import_simple() { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_firefox() { let source = "./test_data/bookmarks_firefox.json"; let temp_dir = tempdir().unwrap(); @@ -61,7 +59,6 @@ fn test_import_firefox() { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_firefox_compressed() { let source = "./test_data/bookmarks_firefox.jsonlz4"; test_utils::create_compressed_bookmarks(Path::new(source)); @@ -73,7 +70,6 @@ fn test_import_firefox_compressed() { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_firefox_bookmark_folder_ubuntu() { let source_path = "./test_data/bookmarks_firefox.jsonlz4"; test_utils::create_compressed_bookmarks(Path::new(source_path)); @@ -91,7 +87,6 @@ fn test_import_firefox_bookmark_folder_ubuntu() { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_firefox_bookmark_folder_macos() { let source_path = "./test_data/bookmarks_firefox.jsonlz4"; test_utils::create_compressed_bookmarks(Path::new(source_path)); @@ -109,7 +104,6 @@ fn test_import_firefox_bookmark_folder_macos() { } #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_chrome() { let source = "./test_data/bookmarks_chromium.json"; let temp_dir = tempdir().unwrap(); @@ -121,7 +115,6 @@ fn test_import_chrome() { // Test renaming of `bookmarks-lock.json` to `bookmarks.json`. #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_import_consecutive() { let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); diff --git a/tests/test_init.rs b/tests/test_init.rs index e3e5698..f5f1d41 100644 --- a/tests/test_init.rs +++ b/tests/test_init.rs @@ -8,7 +8,6 @@ use std::{ use tempfile::tempdir; #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_init() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; diff --git a/tests/test_remove.rs b/tests/test_remove.rs index 3347925..d1d9ec2 100644 --- a/tests/test_remove.rs +++ b/tests/test_remove.rs @@ -4,7 +4,6 @@ use predicates::str; use tempfile::tempdir; #[test] -#[cfg_attr(not(feature = "integration-test"), ignore)] fn test_remove() { let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); diff --git a/tests/test_search.rs b/tests/test_search.rs index 9ff14ea..20ae913 100644 --- a/tests/test_search.rs +++ b/tests/test_search.rs @@ -9,7 +9,6 @@ use std::{ use tempfile::tempdir; #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_search_case_sensitive() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; @@ -67,7 +66,6 @@ async fn test_search_case_sensitive() { } #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_search_case_insensitive() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; @@ -125,7 +123,6 @@ async fn test_search_case_insensitive() { } #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_search_no_content() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; diff --git a/tests/test_update.rs b/tests/test_update.rs index 4cf1199..dab41e0 100644 --- a/tests/test_update.rs +++ b/tests/test_update.rs @@ -8,7 +8,6 @@ use std::{ use tempfile::tempdir; #[tokio::test] -#[cfg_attr(not(feature = "integration-test"), ignore)] async fn test_update() { let mock_server = common::start_mock_server().await; let mocks = common::mount_mocks(&mock_server, 3).await; From 68880d8dcf03a291a3100946681557fb8be8855f Mon Sep 17 00:00:00 2001 From: quambene Date: Sun, 14 Jan 2024 12:22:13 +0100 Subject: [PATCH 7/7] Fix number of processed bookmarks --- CHANGELOG.md | 1 + benches/fetch.rs | 6 +++--- src/cmd/fetch.rs | 18 +++++++++++------- src/cmd/init.rs | 4 ++-- src/cmd/mod.rs | 2 +- src/cmd/update.rs | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b49f762..2dddabf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - changed - Update to rust 1.75 - Fix duplicate cache files for `bogrep fetch --urls` + - Fix report of processed bookmarks - removed - Remove `integration-test` feature diff --git a/benches/fetch.rs b/benches/fetch.rs index c3931e2..fcdade9 100644 --- a/benches/fetch.rs +++ b/benches/fetch.rs @@ -76,7 +76,7 @@ async fn fetch_concurrently(max_concurrent_requests: usize) { let mut bookmarks = TargetBookmarks::new(bookmarks); assert_eq!(bookmarks.len(), 10000); - cmd::fetch_and_cache_bookmarks( + cmd::process_bookmarks( &client, &cache, bookmarks.values_mut().collect(), @@ -124,12 +124,12 @@ async fn fetch_in_parallel(max_parallel_requests: usize) { let client = Arc::new(client); let cache = Arc::new(cache); - fetch_and_cache_bookmarks_in_parallel(client, cache, &bookmarks, max_parallel_requests, true) + process_bookmarks_in_parallel(client, cache, &bookmarks, max_parallel_requests, true) .await .unwrap(); } -pub async fn fetch_and_cache_bookmarks_in_parallel( +pub async fn process_bookmarks_in_parallel( client: Arc, cache: Arc, bookmarks: &[Arc>], diff --git a/src/cmd/fetch.rs b/src/cmd/fetch.rs index ec76e66..3146932 100644 --- a/src/cmd/fetch.rs +++ b/src/cmd/fetch.rs @@ -85,7 +85,7 @@ pub async fn fetch_bookmarks( target_bookmarks.set_action(&Action::FetchAndAdd); } - fetch_and_cache_bookmarks( + process_bookmarks( client, cache, target_bookmarks.values_mut().collect(), @@ -101,13 +101,17 @@ pub async fn fetch_bookmarks( Ok(()) } -/// Fetch all bookmarks and add them to cache. -pub async fn fetch_and_cache_bookmarks( +/// Process bookmarks for all actions except [`Action::None`]. +pub async fn process_bookmarks( client: &impl Fetch, cache: &impl Caching, bookmarks: Vec<&mut TargetBookmark>, max_concurrent_requests: usize, ) -> Result<(), BogrepError> { + let bookmarks = bookmarks + .into_iter() + .filter(|bookmark| bookmark.action != Action::None) + .collect::>(); let mut processed = 0; let mut cached = 0; let mut failed_response = 0; @@ -318,7 +322,7 @@ mod tests { .unwrap(); } - let res = fetch_and_cache_bookmarks( + let res = process_bookmarks( &client, &cache, target_bookmarks.values_mut().collect(), @@ -382,7 +386,7 @@ mod tests { .unwrap(); } - let res = fetch_and_cache_bookmarks( + let res = process_bookmarks( &client, &cache, target_bookmarks.values_mut().collect(), @@ -454,7 +458,7 @@ mod tests { .await .unwrap(); - let res = fetch_and_cache_bookmarks( + let res = process_bookmarks( &client, &cache, target_bookmarks.values_mut().collect(), @@ -528,7 +532,7 @@ mod tests { .await .unwrap(); - let res = fetch_and_cache_bookmarks( + let res = process_bookmarks( &client, &cache, target_bookmarks.values_mut().collect(), diff --git a/src/cmd/init.rs b/src/cmd/init.rs index 669db9f..2ccf817 100644 --- a/src/cmd/init.rs +++ b/src/cmd/init.rs @@ -1,7 +1,7 @@ use crate::{ bookmark_reader::{ReadTarget, SourceReader, WriteTarget}, cache::CacheMode, - cmd::fetch_and_cache_bookmarks, + cmd::process_bookmarks, utils, Action, Cache, Caching, Client, Config, Fetch, InitArgs, SourceBookmarks, TargetBookmarks, }; @@ -75,7 +75,7 @@ async fn init_bookmarks( .join(", ") ); - fetch_and_cache_bookmarks( + process_bookmarks( client, cache, target_bookmarks.values_mut().collect(), diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index f6e4a72..4e2467b 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -11,7 +11,7 @@ mod update; pub use add::add; pub use clean::clean; pub use configure::configure; -pub use fetch::{fetch, fetch_and_cache_bookmarks, fetch_diff}; +pub use fetch::{fetch, fetch_diff, process_bookmarks}; pub use import::import; pub use init::init; pub use remove::remove; diff --git a/src/cmd/update.rs b/src/cmd/update.rs index f77875d..522e21f 100644 --- a/src/cmd/update.rs +++ b/src/cmd/update.rs @@ -60,7 +60,7 @@ async fn update_bookmarks( target_bookmarks.update(&source_bookmarks)?; - cmd::fetch_and_cache_bookmarks( + cmd::process_bookmarks( client, cache, target_bookmarks.values_mut().collect(),