From 9cc37046790e64488171a617088785f3e14bfdfe Mon Sep 17 00:00:00 2001 From: Brendon Walsh Date: Sat, 11 Nov 2023 22:37:05 -0500 Subject: [PATCH] Ignore paths in the scanner with `.ignore` or `.plexignore` files (#574) * ignore::WalkBuilder to support .plexignore semantics * fixup rebase conflicts --- Cargo.lock | 54 +++++++++++++++++++++++++++---- dim-core/Cargo.toml | 2 +- dim-core/src/scanner/mod.rs | 64 ++++++++++++++++++++++++++++++++++--- dim-web/src/lib.rs | 2 +- 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47204cc9d..7d363fcff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -372,6 +372,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -859,6 +869,7 @@ dependencies = [ "fuzzy-matcher", "http", "hyper", + "ignore", "image", "itertools", "lazy_static", @@ -889,7 +900,6 @@ dependencies = [ "tracing-subscriber", "url", "uuid 1.5.0", - "walkdir", "warp", "xmlwriter", "xtra", @@ -1044,9 +1054,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -1310,9 +1320,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -1345,6 +1355,19 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "globset" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + [[package]] name = "governor" version = "0.5.1" @@ -1592,6 +1615,23 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" +dependencies = [ + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "image" version = "0.24.7" @@ -1771,9 +1811,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" diff --git a/dim-core/Cargo.toml b/dim-core/Cargo.toml index 56e4e64c6..4495a36e5 100644 --- a/dim-core/Cargo.toml +++ b/dim-core/Cargo.toml @@ -46,6 +46,7 @@ futures = "0.3.14" fuzzy-matcher = "0.3.7" http = "^0.2.3" hyper = "0.14.20" +ignore = "0.4.20" image = "0.24.3" itertools = "0.10.3" lazy_static = "1.4.0" @@ -77,7 +78,6 @@ tracing-subscriber = { version = "^0.3.10", features = [ ] } url = "2.2.2" uuid = { version = "1.2.2", features = ["v4"] } -walkdir = "2.3.1" warp = { version = "0.3.3", features = ["tls", "tokio-rustls"] } xmlwriter = "0.1.0" xtra = { version = "0.5.1", features = ["tokio", "with-tokio-1"] } diff --git a/dim-core/src/scanner/mod.rs b/dim-core/src/scanner/mod.rs index f64bc9f3f..e0245e469 100644 --- a/dim-core/src/scanner/mod.rs +++ b/dim-core/src/scanner/mod.rs @@ -27,6 +27,7 @@ use dim_extern_api::filename::TorrentMetadata; use dim_extern_api::ExternalQueryIntoShow; use futures::FutureExt; +use ignore::WalkBuilder; use itertools::Itertools; use std::ffi::OsStr; @@ -41,11 +42,65 @@ use tracing::error; use tracing::info; use tracing::instrument; use tracing::warn; -use walkdir::WalkDir; pub use error::Error; -pub(super) static SUPPORTED_EXTS: &[&str] = &["mp4", "mkv", "avi", "webm"]; +pub(super) static SUPPORTED_EXTS: &[&str] = &[ + "001", + "3g2", + "3gp", + "amv", + "asf", + "asx", + "avi", + "bin", + "bivx", + "divx", + "dv", + "dvr-ms", + "f4v", + "fli", + "flv", + "ifo", + "img", + "iso", + "m2t", + "m2ts", + "m2v", + "m4v", + "mkv", + "mk3d", + "mov", + "mp4", + "mpe", + "mpeg", + "mpg", + "mts", + "mxf", + "nrg", + "nsv", + "nuv", + "ogg", + "ogm", + "ogv", + "pva", + "qt", + "rec", + "rm", + "rmvb", + "strm", + "svq3", + "tp", + "ts", + "ty", + "viv", + "vob", + "vp3", + "webm", + "wmv", + "wtv", + "xvid" +]; /// Function recursively walks the paths passed and returns all files in those directories. /// FIXME: THIS IS NOT ASYNC-SAFE!!! @@ -55,10 +110,11 @@ pub(super) static SUPPORTED_EXTS: &[&str] = &["mp4", "mkv", "avi", "webm"]; pub fn get_subfiles(paths: impl Iterator>) -> Vec { let mut files = Vec::with_capacity(2048); for path in paths { - let mut subfiles: Vec = WalkDir::new(path) + let mut subfiles = WalkBuilder::new(path) // we want to follow all symlinks in case of complex dir structures .follow_links(true) - .into_iter() + .add_custom_ignore_filename(".plexignore") + .build() .filter_map(Result::ok) // ignore all hidden files. .filter(|f| { diff --git a/dim-web/src/lib.rs b/dim-web/src/lib.rs index d04529753..39d78d18c 100644 --- a/dim-web/src/lib.rs +++ b/dim-web/src/lib.rs @@ -40,7 +40,7 @@ pub struct AppState { fn library_routes(app: AppState) -> Router { Router::new() .route( - "api/v1/library", + "/api/v1/library", post(routes::library::library_post).get(routes::library::library_get_all), ) .route(