move external module to dim-extern-api (#541)

Cargo.toml

@@ -1,7 +1,5 @@
-cargo-features = ["workspace-inheritance"]
-
 [workspace]
-members = ["dim", "dim-auth", "dim-database", "dim-events"]
+members = ["dim", "dim-auth", "dim-database", "dim-events", "dim-extern-api"]
 
 [workspace.package]
 version = "0.4.0-dev"

dim-extern-api/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "dim-extern-api"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+anitomy = "0.1.2"
+async-trait = "0.1.68"
+chrono = { version = "0.4.24", features = ["serde"] }
+dashmap = "5.4.0"
+displaydoc = "0.2.3"
+governor = "0.5.1"
+rand = { version = "0.8.5", features = ["small_rng"] }
+reqwest = { version = "0.11.0", features = ["json", "rustls-tls", "brotli"], default-features = false }
+retry-block = "1.0.0"
+serde = { version = "1.0.159", features = ["derive", "rc"] }
+serde_json = "1.0.95"
+thiserror = "1.0.40"
+tokio = { version = "1.27.0", features = ["sync", "rt"] }
+torrent-name-parser = "0.12.0"
+tracing = "0.1.37"

dim/src/external/filename.rs → dim-extern-api/src/filename.rs

@@ -1,6 +1,6 @@
-use anitomy::Anitomy;
+pub use anitomy::Anitomy;
 use anitomy::ElementCategory;
-use torrent_name_parser::Metadata as TorrentMetadata;
+pub use torrent_name_parser::Metadata as TorrentMetadata;
 
 #[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub struct Metadata {

dim/src/external/mod.rs → dim-extern-api/src/lib.rs

@@ -1,6 +1,9 @@
-/// Module contains a common interface for extracting and obtaining filename metadata.
+//! Library contains a common interface for extracting and obtaining filename metadata as well as
+//! the implementations for various external APIs, such as TMDB.
+
 pub mod filename;
 pub mod mock;
+pub mod tmdb;
 
 use async_trait::async_trait;
 
@@ -13,8 +16,6 @@ use serde::Deserialize;
 use serde::Serialize;
 use thiserror::Error;
 
-pub mod tmdb;
-
 pub type Result<T> = ::core::result::Result<T, Error>;
 
 #[derive(Clone, Display, Debug, Error, Serialize)]

dim/src/external/mock.rs → dim-extern-api/src/mock.rs

@@ -1,7 +1,7 @@
-use super::ExternalActor;
-use super::ExternalMedia;
-use super::ExternalQuery;
-use super::Result;
+use crate::ExternalActor;
+use crate::ExternalMedia;
+use crate::ExternalQuery;
+use crate::Result;
 
 #[derive(Debug, Clone, Copy)]
 pub struct MockProvider;

dim/src/external/tmdb/cache_control.rs → dim-extern-api/src/tmdb/cache_control.rs

@@ -1,4 +1,4 @@
-use crate::external::MediaSearchType;
+use crate::MediaSearchType;
 
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;

dim/src/external/tmdb/metadata_provider.rs → dim-extern-api/src/tmdb/metadata_provider.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 use std::future::Future;
 use std::marker::PhantomData;
 use std::num::NonZeroU32;
+use std::result::Result;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
 use std::time::Duration;
@@ -12,8 +13,8 @@ use async_trait::async_trait;
 use tokio::sync::broadcast;
 use tracing::instrument;
 
-use crate::external::{Result as QueryResult, *};
-use core::result::Result;
+use crate::Result as QueryResult;
+use crate::*;
 
 use governor::clock::DefaultClock;
 use governor::middleware::NoOpMiddleware;
@@ -246,7 +247,7 @@ impl TMDBMetadataProvider {
             .await?;
 
         let mut search = serde_json::from_str::<SearchResponse>(&st).map_err(|error| {
-            crate::external::Error::DeserializationError {
+            Error::DeserializationError {
                 body: st,
                 error: format!("{error}"),
             }
@@ -264,7 +265,7 @@ impl TMDBMetadataProvider {
                 .await?;
 
             let genre_list = serde_json::from_str::<GenreList>(&st).map_err(|error| {
-                crate::external::Error::DeserializationError {
+                Error::DeserializationError {
                     body: st,
                     error: format!("{error}"),
                 }
@@ -335,7 +336,7 @@ impl TMDBMetadataProvider {
             .await?;
 
         let details = serde_json::from_str::<TMDBMediaObject>(&response_body).map_err(|err| {
-            crate::external::Error::DeserializationError {
+            Error::DeserializationError {
                 body: response_body,
                 error: format!("{err}"),
             }
@@ -367,12 +368,11 @@ impl TMDBMetadataProvider {
             )
             .await?;
 
-        let actor = serde_json::from_str::<Cast>(&resp).map_err(|error| {
-            crate::external::Error::DeserializationError {
+        let actor =
+            serde_json::from_str::<Cast>(&resp).map_err(|error| Error::DeserializationError {
                 body: resp,
                 error: format!("{error}"),
-            }
-        })?;
+            })?;
 
         Ok(actor.cast.into_iter().map(|x| x.into()).collect())
     }
@@ -400,7 +400,7 @@ impl TMDBMetadataProvider {
             .await?;
 
         let tv_details = serde_json::from_str::<TvSeasons>(&response_body).map_err(|err| {
-            crate::external::Error::DeserializationError {
+            Error::DeserializationError {
                 body: response_body,
                 error: format!("{err}"),
             }
@@ -434,7 +434,7 @@ impl TMDBMetadataProvider {
             .await?;
 
         let tv_details = serde_json::from_str::<TvEpisodes>(&response_body).map_err(|err| {
-            crate::external::Error::DeserializationError {
+            Error::DeserializationError {
                 body: response_body,
                 error: format!("{err}"),
             }

dim/src/external/tmdb/mod.rs → dim-extern-api/src/tmdb/mod.rs

@@ -69,9 +69,7 @@ mod tests {
     use chrono::{Datelike, Timelike};
 
     use super::*;
-    use crate::external::{
-        ExternalEpisode, ExternalMedia, ExternalQuery, ExternalQueryShow, ExternalSeason,
-    };
+    use crate::{ExternalEpisode, ExternalMedia, ExternalQuery, ExternalQueryShow, ExternalSeason};
 
     fn make_letterkenny() -> ExternalMedia {
         let dt = chrono::Utc::now()

dim/src/external/tmdb/raw_client.rs → dim-extern-api/src/tmdb/raw_client.rs

@@ -6,9 +6,7 @@ use serde::Deserialize;
 use std::future::Future;
 use std::time::Duration;
 
-use crate::external::{
-    ExternalActor, ExternalEpisode, ExternalMedia, ExternalSeason, MediaSearchType,
-};
+use crate::{ExternalActor, ExternalEpisode, ExternalMedia, ExternalSeason, MediaSearchType};
 
 use super::{TMDBClientRequestError, TMDBMetadataProvider, TMDB_BASE_URL};
 

dim/Cargo.toml

@@ -21,24 +21,21 @@ nightfall = { git = "https://github.com/Dusk-Labs/nightfall", tag = "0.3.12-rc4"
 dim-auth = { path = "../dim-auth" }
 dim-database = { path = "../dim-database" }
 dim-events = { path = "../dim-events" }
+dim-extern-api = { path = "../dim-extern-api" }
 
 serde = { version = "^1.0.125", default-features = false, features = ["derive", "std", "rc"] }
 serde_derive = "^1.0.125"
 serde_json = "^1.0.64"
 
-anitomy = "0.1.2"
-async-recursion = "0.3.2"
 async-trait = "0.1.50"
 bytes = "1.0.1"
 cfg-if = "1.0.0"
 chrono = { version = "0.4.19", features = ["serde"] }
-dashmap = "5.3.3"
 dia-i18n = "0.10.0"
 displaydoc = "0.2.3"
 dominant_color = "0.3.0"
 futures = "0.3.14"
 fuzzy-matcher = "0.3.7"
-governor = "0.5.0"
 http = "^0.2.3"
 image = "0.24.3"
 itertools = "0.10.3"
@@ -47,7 +44,6 @@ notify = "5.0.0"
 once_cell = "1.8.0"
 parking_lot = "0.12.0"
 percent-encoding = "2.1.0"
-priority-queue = "1.2.0"
 rand = { version = "0.8.5", features = ["small_rng"] }
 reqwest = { version = "0.11.0", features = ["json", "rustls-tls", "brotli"], default-features = false }
 rust-embed = "^5.9.0"
@@ -56,7 +52,6 @@ structopt = "0.3.21"
 thiserror = "1.0.30"
 tokio = { version = "1", features = ["rt", "signal", "full", "tracing"] }
 toml = "0.5.8"
-torrent-name-parser = "0.6.3"
 tracing = "0.1.32"
 tracing-appender = "0.2.0"
 tracing-subscriber = { version = "^0.3.10", features = ["fmt", "env-filter", "json"] }
@@ -70,7 +65,6 @@ xtra = { version = "0.5.1", features = ["with-tokio-1"] }
 # FIXME: Remove when we get rid of xtra_proc
 new_xtra = { package = "xtra", git = "https://github.com/Restioson/xtra", features = ["with-tokio-1"] }
 url = "2.2.2"
-retry-block = "1.0.0"
 hyper = "0.14.20"
 rusqlite = { version = "0.27.0", features = ["hooks"] }
 libsqlite3-sys = { version = "^0.24.0" }

dim/src/core.rs

@@ -1,5 +1,4 @@
 use crate::balanced_or_tree;
-use crate::external::tmdb::TMDBMetadataProvider;
 use crate::logger::RequestLogger;
 use crate::routes;
 use crate::routes::*;
@@ -8,6 +7,7 @@ use crate::stream_tracking::StreamTracking;
 use crate::websocket;
 
 use dim_database::library::MediaType;
+use dim_extern_api::tmdb::TMDBMetadataProvider;
 
 use once_cell::sync::OnceCell;
 

dim/src/errors.rs

@@ -63,7 +63,7 @@ pub enum DimError {
     /// User does not exist
     UserNotFound,
     /// Couldn't find the tmdb id provided.
-    ExternalSearchError(crate::external::Error),
+    ExternalSearchError(dim_extern_api::Error),
 }
 
 impl From<sqlx::Error> for DimError {

dim/src/lib.rs

@@ -29,8 +29,6 @@ use tracing_subscriber::EnvFilter;
 pub mod core;
 /// Module contains all the error definitions used in dim, and returned by the web-service.
 pub mod errors;
-/// Module contains our external api interfaces
-pub mod external;
 /// Contains the code for fetching assets like posters and stills.
 pub mod fetcher;
 /// Inspect api for Result type

dim/src/routes/library.rs

@@ -1,7 +1,6 @@
 use crate::core::DbConnection;
 use crate::core::EventTx;
 use crate::errors;
-use crate::external::tmdb::TMDBMetadataProvider;
 use crate::json;
 use crate::scanner;
 use crate::scanner::daemon::FsWatcher;
@@ -13,9 +12,10 @@ use dim_database::library::Library;
 use dim_database::library::MediaType;
 use dim_database::media::Media;
 use dim_database::mediafile::MediaFile;
-
 use dim_database::user::User;
 
+use dim_extern_api::tmdb::TMDBMetadataProvider;
+
 use std::collections::HashMap;
 use std::sync::Arc;
 

dim/src/routes/media.rs

@@ -1,13 +1,10 @@
 use crate::core::DbConnection;
 use crate::errors;
-use crate::external::tmdb::TMDBMetadataProvider;
-use crate::external::ExternalQueryIntoShow;
 use crate::json;
 use crate::tree;
 use crate::utils::secs_to_pretty;
 
 use chrono::Datelike;
-use dim_database::user::User;
 
 use dim_database::compact_mediafile::CompactMediafile;
 use dim_database::episode::Episode;
@@ -17,6 +14,10 @@ use dim_database::media::Media;
 use dim_database::media::UpdateMedia;
 use dim_database::mediafile::MediaFile;
 use dim_database::progress::Progress;
+use dim_database::user::User;
+
+use dim_extern_api::tmdb::TMDBMetadataProvider;
+use dim_extern_api::ExternalQueryIntoShow;
 
 use warp::http::status::StatusCode;
 use warp::reply;

dim/src/routes/mediafile.rs

@@ -1,7 +1,6 @@
 use crate::core::DbConnection;
 use crate::errors;
 use crate::errors::ErrorStatusCode;
-use crate::external::ExternalQueryIntoShow;
 use crate::scanner::movie;
 use crate::scanner::parse_filenames;
 use crate::scanner::tv_show;
@@ -15,6 +14,8 @@ use dim_database::library::MediaType;
 use dim_database::mediafile::MediaFile;
 use dim_database::user::User;
 
+use dim_extern_api::ExternalQueryIntoShow;
+
 use serde::Serialize;
 use serde_json::json;
 use std::sync::Arc;

dim/src/routes/rematch_media.rs

@@ -1,7 +1,6 @@
 use crate::core::DbConnection;
 use crate::core::EventTx;
 use crate::errors::*;
-use crate::external::ExternalQueryIntoShow;
 use crate::scanner::movie;
 use crate::scanner::parse_filenames;
 use crate::scanner::tv_show;
@@ -16,6 +15,8 @@ use std::sync::Arc;
 use dim_database::library::MediaType;
 use dim_database::mediafile::MediaFile;
 
+use dim_extern_api::ExternalQueryIntoShow;
+
 use tracing::error;
 use tracing::info;
 

dim/src/scanner/daemon.rs

@@ -1,5 +1,5 @@
 use crate::core::EventTx;
-use crate::external::ExternalQueryIntoShow;
+use dim_extern_api::ExternalQueryIntoShow;
 
 use super::movie;
 use super::tv_show;

dim/src/scanner/mediafile.rs

@@ -1,8 +1,8 @@
 //! Module contains all the code that creates and inserts basic mediafiles into the database.
 
-use crate::external::filename::Metadata;
 use crate::streaming::ffprobe::FFProbeCtx;
 use crate::streaming::FFPROBE_BIN;
+use dim_extern_api::filename::Metadata;
 
 use async_trait::async_trait;
 

dim/src/scanner/mod.rs

@@ -10,20 +10,22 @@ pub mod tv_show;
 
 use self::mediafile::Error as CreatorError;
 use self::mediafile::MediafileCreator;
-use super::external::filename::CombinedExtractor;
-use super::external::filename::FilenameMetadata;
-use super::external::filename::Metadata;
 use crate::core::EventTx;
-use crate::external::ExternalQueryIntoShow;
 
-use anitomy::Anitomy;
 use async_trait::async_trait;
 
 use dim_database::library::Library;
 use dim_database::library::MediaType;
 use dim_database::mediafile::InsertableMediaFile;
 use dim_database::mediafile::MediaFile;
 
+use dim_extern_api::filename::Anitomy;
+use dim_extern_api::filename::CombinedExtractor;
+use dim_extern_api::filename::FilenameMetadata;
+use dim_extern_api::filename::Metadata;
+use dim_extern_api::filename::TorrentMetadata;
+use dim_extern_api::ExternalQueryIntoShow;
+
 use futures::FutureExt;
 use itertools::Itertools;
 
@@ -35,7 +37,6 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Instant;
 
-use torrent_name_parser::Metadata as TorrentMetadata;
 use tracing::error;
 use tracing::info;
 use tracing::instrument;