From ebee1f4a6ce690ffcae12ad3de4ce1848ad131fc Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Sep 2024 20:57:50 +0530 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9E=95=20build(deps):=20add=20`itertools?= =?UTF-8?q?`=20crate=20(#592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fa2aebdb..8731e4d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,14 +82,13 @@ base64 = { version = "0.21.5", default-features = false, features = [ cfg-if = { version = "1.0.0", default-features = false, optional = true } keyword_extraction = { version = "1.4.3", default-features = false, features = [ "tf_idf", - - ] } stop-words = { version = "0.8.0", default-features = false, features = ["iso"] } thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [ "moby", -] } +]} +itertools = {version = "0.13.0", default-features = false} [dev-dependencies] rusty-hook = { version = "^0.11.2", default-features = false } From 9a5f1c5f449bc96d1c4c1650aba2bf23b1b22852 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Sep 2024 21:10:54 +0530 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20reimplement=20caching?= =?UTF-8?q?=20code=20within=20the=20`search`=20function=20(#592)=20-=20red?= =?UTF-8?q?uce=20resource=20usage=20&=20-=20only=20cache=20search=20result?= =?UTF-8?q?s=20which=20has=20not=20been=20cached=20before.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes/search.rs | 48 ++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 103b5812..b25077ce 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -12,6 +12,7 @@ use crate::{ results::aggregator::aggregate, }; use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse}; +use itertools::Itertools; use regex::Regex; use std::borrow::Cow; use tokio::{ @@ -40,7 +41,6 @@ pub async fn search( config: web::Data<&'static Config>, cache: web::Data<&'static SharedCache>, ) -> Result> { - use std::sync::Arc; let params = web::Query::::from_query(req.query_string())?; match ¶ms.q { Some(query) => { @@ -83,44 +83,36 @@ pub async fn search( let previous_page = page.saturating_sub(1); let next_page = page + 1; - let mut results = Arc::new((SearchResults::default(), String::default())); + let results: (SearchResults, String, bool); if page != previous_page { let (previous_results, current_results, next_results) = join!( get_results(previous_page), get_results(page), get_results(next_page) ); - let (parsed_previous_results, parsed_next_results) = - (previous_results?, next_results?); - let (cache_keys, results_list) = ( - [ - parsed_previous_results.1, - results.1.clone(), - parsed_next_results.1, - ], - [ - parsed_previous_results.0, - results.0.clone(), - parsed_next_results.0, - ], - ); + results = current_results?; - results = Arc::new(current_results?); + let (results_list, cache_keys): (Vec, Vec) = + [previous_results?, results.clone(), next_results?] + .into_iter() + .filter_map(|(result, cache_key, flag)| { + dbg!(flag).then_some((result, cache_key)) + }) + .multiunzip(); tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); } else { let (current_results, next_results) = join!(get_results(page), get_results(page + 1)); - let parsed_next_results = next_results?; - - results = Arc::new(current_results?); + results = current_results?; - let (cache_keys, results_list) = ( - [results.1.clone(), parsed_next_results.1.clone()], - [results.0.clone(), parsed_next_results.0], - ); + let (results_list, cache_keys): (Vec, Vec) = + [results.clone(), next_results?] + .into_iter() + .filter_map(|(result, cache_key, flag)| flag.then_some((result, cache_key))) + .multiunzip(); tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); } @@ -163,7 +155,7 @@ async fn results( query: &str, page: u32, search_settings: &server_models::Cookie<'_>, -) -> Result<(SearchResults, String), Box> { +) -> Result<(SearchResults, String, bool), Box> { // eagerly parse cookie value to evaluate safe search level let safe_search_level = search_settings.safe_search_level; @@ -182,7 +174,7 @@ async fn results( // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results { - Ok(results) => Ok((results, cache_key)), + Ok(results) => Ok((results, cache_key, false)), Err(_) => { if safe_search_level == 4 { let mut results: SearchResults = SearchResults::default(); @@ -196,7 +188,7 @@ async fn results( .cache_results(&[results.clone()], &[cache_key.clone()]) .await?; results.set_safe_search_level(safe_search_level); - return Ok((results, cache_key)); + return Ok((results, cache_key, true)); } } @@ -235,7 +227,7 @@ async fn results( .cache_results(&[results.clone()], &[cache_key.clone()]) .await?; results.set_safe_search_level(safe_search_level); - Ok((results, cache_key)) + Ok((results, cache_key, true)) } } } From acee5d892d9997fb9c69346ae2da69a27203e21c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Sep 2024 21:17:31 +0530 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20replace=20deprecated?= =?UTF-8?q?=20`set=5Fex`=20command=20with=20`set=5Foptions`=20in=20`cache?= =?UTF-8?q?=5Fjson`=20function=20(#592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/redis_cacher.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/cache/redis_cacher.rs b/src/cache/redis_cacher.rs index 1dd7cb82..1fde95b9 100644 --- a/src/cache/redis_cacher.rs +++ b/src/cache/redis_cacher.rs @@ -4,7 +4,10 @@ use super::error::CacheError; use error_stack::Report; use futures::stream::FuturesUnordered; -use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError}; +use redis::{ + aio::ConnectionManager, AsyncCommands, Client, ExistenceCheck, RedisError, SetExpiry, + SetOptions, +}; /// A constant holding the redis pipeline size. const REDIS_PIPELINE_SIZE: usize = 3; @@ -139,8 +142,14 @@ impl RedisCache { self.current_connection = Default::default(); for (key, json_result) in keys.zip(json_results) { - self.pipeline - .set_ex(key, json_result, self.cache_ttl.into()); + self.pipeline.set_options( + key, + json_result, + SetOptions::default() + .conditional_set(ExistenceCheck::NX) + .get(true) + .with_expiration(SetExpiry::EX(self.cache_ttl.into())), + ); } let mut result: Result<(), RedisError> = self From b22d60f166b868ea1ccd42e744aa6383130e899b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Sep 2024 21:40:49 +0530 Subject: [PATCH 4/6] :bookmark: chore(release): bump the app version (#592) --- Cargo.lock | 43 ++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9967ec69..30f4dc7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,7 +351,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ - "brotli 5.0.0", + "brotli", "flate2", "futures-core", "memchr", @@ -490,28 +490,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - -[[package]] -name = "brotli" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -872,7 +851,7 @@ dependencies = [ "clap", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -891,7 +870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -1902,6 +1881,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -2004,7 +1992,7 @@ dependencies = [ "cssparser-color", "data-encoding", "getrandom", - "itertools", + "itertools 0.10.5", "lazy_static", "parcel_selectors", "paste", @@ -4461,7 +4449,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.17.0" +version = "1.17.20" dependencies = [ "actix-cors", "actix-files", @@ -4481,6 +4469,7 @@ dependencies = [ "error-stack", "fake-useragent", "futures 0.3.30", + "itertools 0.13.0", "keyword_extraction", "lightningcss", "log", diff --git a/Cargo.toml b/Cargo.toml index 8731e4d5..f0481b2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.17.0" +version = "1.17.20" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 3a1ff0f307c1ad99fd5661fb9c0acf11a998108f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:17:18 +0000 Subject: [PATCH 5/6] build(deps): bump quinn-proto from 0.11.6 to 0.11.8 (#600) --- Cargo.lock | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9967ec69..6a68778b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,7 +351,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ - "brotli 5.0.0", + "brotli", "flate2", "futures-core", "memchr", @@ -490,28 +490,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - -[[package]] -name = "brotli" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -2849,9 +2828,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes 1.6.0", "rand 0.8.5", From 9984ba0d12ff7d141cb340cdf86124d769890b9d Mon Sep 17 00:00:00 2001 From: Jann Marc Villablanca <31008330+jfvillablanca@users.noreply.github.com> Date: Fri, 6 Sep 2024 13:08:41 +0800 Subject: [PATCH 6/6] :bug: fix: update nix flake lock (#602) - this updates rustc to 1.80.1 - `eslint` has been moved as a top-level package in upstream NixOS/nixpkgs Context: package maintainers are slowly de-bloating `node-packages.nix` Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- flake.lock | 6 +++--- flake.nix | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 2d1b886c..da3efdbc 100644 --- a/flake.lock +++ b/flake.lock @@ -34,11 +34,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1695318763, - "narHash": "sha256-FHVPDRP2AfvsxAdc+AsgFJevMz5VBmnZglFUMlxBkcY=", + "lastModified": 1725194671, + "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e12483116b3b51a185a33a272bf351e357ba9a99", + "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 6c415142..e626d592 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ haskellPackages.hadolint nodejs nodePackages_latest.cspell - nodePackages_latest.eslint + eslint nodePackages_latest.markdownlint-cli2 nodePackages_latest.stylelint redis