diff --git a/.github/workflows/build-check.yml b/.github/workflows/build-check.yml index 34ea2c98a..e09be7c18 100644 --- a/.github/workflows/build-check.yml +++ b/.github/workflows/build-check.yml @@ -90,4 +90,4 @@ jobs: target/x86_64-unknown-linux-musl/release/hot-or-not-web-leptos-ssr target/release/hash.txt target/site - .empty \ No newline at end of file + .empty diff --git a/Cargo.lock b/Cargo.lock index e65a2197b..d444ee889 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2611,7 +2611,8 @@ dependencies = [ "log", "once_cell", "openidconnect", - "prost", + "prost 0.12.6", + "prost 0.13.1", "rand_chacha", "redb", "redis", @@ -2624,8 +2625,11 @@ dependencies = [ "testcontainers", "thiserror", "tokio", - "tonic", - "tonic-build", + "tonic 0.11.0", + "tonic 0.12.1", + "tonic-build 0.11.0", + "tonic-build 0.12.0", + "tonic-web-wasm-client", "tower", "tower-http", "tracing", @@ -4529,7 +4533,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +dependencies = [ + "bytes", + "prost-derive 0.13.1", ] [[package]] @@ -4546,8 +4560,29 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn 2.0.72", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.13.1", + "prost-types 0.13.1", "regex", "syn 2.0.72", "tempfile", @@ -4566,13 +4601,35 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "prost-derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", +] + +[[package]] +name = "prost-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +dependencies = [ + "prost 0.13.1", ] [[package]] @@ -6049,7 +6106,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.6", "rustls-pemfile 2.1.2", "rustls-pki-types", "tokio", @@ -6062,6 +6119,27 @@ dependencies = [ "webpki-roots 0.26.3", ] +[[package]] +name = "tonic" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project", + "prost 0.13.1", + "tokio-stream", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.11.0" @@ -6070,11 +6148,49 @@ checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", + "prost-build 0.12.6", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tonic-build" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "690943cc223adcdd67bb597a2e573ead1b88e999ba37528fe8e6356bf44b29b6" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.13.1", "quote", "syn 2.0.72", ] +[[package]] +name = "tonic-web-wasm-client" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5ca6e7bdd0042c440d36b6df97c1436f1d45871ce18298091f114004b1beb4" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "httparse", + "js-sys", + "pin-project", + "thiserror", + "tonic 0.12.1", + "tower-service", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 37731458f..765d1c63a 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -16,7 +16,10 @@ leptos_meta = { version = "0.6", features = ["nightly"] } leptos_router = { version = "0.6", features = ["nightly"] } log = "0.4" simple_logger = "4.0" -tokio = { version = "1", optional = true, features = ["rt-multi-thread", "signal"] } +tokio = { version = "1", optional = true, features = [ + "rt-multi-thread", + "signal", +] } tower = { version = "0.4", optional = true } tower-http = { version = "0.5", features = ["fs"], optional = true } wasm-bindgen = "=0.2.92" @@ -86,12 +89,27 @@ gloo-utils = { version = "0.2.0", features = ["serde"] } tonic = { version = "0.11.0", features = [ "tls", "tls-webpki-roots", + "transport", # for clippy ], optional = true } prost = { version = "0.12.4", optional = true } hmac = { version = "0.12.1", optional = true } wasm-bindgen-futures = { version = "0.4.42", optional = true } testcontainers = { version = "0.20.0", optional = true } yral-testcontainers = { git = "https://github.com/go-bazzinga/yral-testcontainers", rev = "f9d2c01c498d58fca0595a48bdc3f9400e57ec2f", optional = true } +tonic-web-wasm-client = { version = "0.6" } + +[dependencies.tonic_2] +package = "tonic" +version = "0.12.0" +optional = true +default-features = false +features = ["prost", "codegen"] + + +[dependencies.prost_2] +package = "prost" +version = "0.13.0" +optional = true [build-dependencies] serde = { version = "1.0", features = ["derive"] } @@ -101,6 +119,12 @@ convert_case = "0.6.0" tonic-build = "0.11.0" anyhow = "1.0.86" +[build-dependencies.tonic_build_2] +package = "tonic-build" +version = "=0.12.0" +default-features = false +features = ["prost"] + [features] hydrate = [ "leptos/hydrate", @@ -110,7 +134,9 @@ hydrate = [ "dep:web-sys", "reqwest/native-tls", "dep:rand_chacha", - "dep:wasm-bindgen-futures" + "dep:wasm-bindgen-futures", + "tonic_2", + "prost_2", ] ssr = [ "dep:axum", @@ -150,6 +176,7 @@ redis-kv = [] cloudflare = ["dep:gob-cloudflare"] backend-admin = [] ga4 = [] +local-feed = [] mock-wallet-history = ["dep:rand_chacha"] release-bin = [ "ssr", @@ -176,12 +203,7 @@ local-bin = [ "dep:testcontainers", "dep:yral-testcontainers", ] -local-lib = [ - "hydrate", - "redis-kv", - "local-auth", - "backend-admin", -] +local-lib = ["hydrate", "redis-kv", "local-auth", "backend-admin"] [package.metadata.leptos] # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name @@ -238,7 +260,7 @@ env = "DEV" # The features to use when compiling the bin target # # Optional. Can be over-ridden with the command line parameter --bin-features -bin-features = ["ssr","local-auth"] +bin-features = ["ssr", "local-auth"] # If the --no-default-features flag should be used when compiling the bin target # diff --git a/ssr/build.rs b/ssr/build.rs index c631bd820..4bc3f4689 100644 --- a/ssr/build.rs +++ b/ssr/build.rs @@ -126,8 +126,33 @@ mod build_common { Ok(()) } + fn build_gprc_client() -> Result<()> { + let ml_feed_proto = "contracts/projects/ml_feed/ml_feed.proto"; + let mut out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + tonic_build::configure() + .build_client(true) + .build_server(false) + .out_dir(out_dir.clone()) + .compile(&[ml_feed_proto], &["proto"])?; + + out_dir = out_dir.join("grpc-web"); + fs::create_dir_all(&out_dir)?; + + tonic_build_2::configure() + .build_client(true) + .build_server(false) + .out_dir(out_dir) + .compile(&[ml_feed_proto], &["proto"])?; + + Ok(()) + } + pub fn build_common() -> Result<()> { build_did_intf()?; + + build_gprc_client()?; + Ok(()) } } diff --git a/ssr/contracts b/ssr/contracts index dd72cc15f..3e8f5911f 160000 --- a/ssr/contracts +++ b/ssr/contracts @@ -1 +1 @@ -Subproject commit dd72cc15f9d87d7d5b9e51bfe2c77f06971f98a4 +Subproject commit 3e8f5911f9fba6746121b7847243d5d0da3748ee diff --git a/ssr/src/app.rs b/ssr/src/app.rs index 6c38fddb4..82113bc71 100644 --- a/ssr/src/app.rs +++ b/ssr/src/app.rs @@ -26,7 +26,7 @@ use leptos_router::*; fn NotFound() -> impl IntoView { let mut outside_errors = Errors::default(); outside_errors.insert_with_default_key(AppError::NotFound); - view! { } + view! { } } #[component(transparent)] @@ -35,11 +35,11 @@ fn GoogleAuthRedirectHandlerRoute() -> impl IntoView { #[cfg(any(feature = "oauth-ssr", feature = "oauth-hydrate"))] { use crate::page::google_redirect::GoogleRedirectHandler; - view! { } + view! { } } #[cfg(not(any(feature = "oauth-ssr", feature = "oauth-hydrate")))] { - view! { } + view! { } } } @@ -49,11 +49,11 @@ fn GoogleAuthRedirectorRoute() -> impl IntoView { #[cfg(any(feature = "oauth-ssr", feature = "oauth-hydrate"))] { use crate::page::google_redirect::GoogleRedirector; - view! { } + view! { } } #[cfg(not(any(feature = "oauth-ssr", feature = "oauth-hydrate")))] { - view! { } + view! { } } } @@ -67,6 +67,12 @@ pub fn App() -> impl IntoView { provide_context(ProfilePostsContext::default()); provide_context(AuthorizedUserToSeedContent::default()); + #[cfg(feature = "hydrate")] + { + use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; + provide_context(MLFeed::default()); + } + // History Tracking let history_ctx = HistoryCtx::default(); provide_context(history_ctx.clone()); @@ -85,12 +91,12 @@ pub fn App() -> impl IntoView { } view! { - + // sets the document title - + <Title text="Yral" /> - <Link rel="manifest" href="/app.webmanifest"/> + <Link rel="manifest" href="/app.webmanifest" /> // GA4 Global Site Tag (gtag.js) - Google Analytics // G-6W5Q2MRX0E to test locally | G-PLNNETMSLM @@ -110,36 +116,36 @@ pub fn App() -> impl IntoView { </Show> // content for this welcome page - <Router fallback=|| view! { <NotFound/> }.into_view()> + <Router fallback=|| view! { <NotFound /> }.into_view()> <main> <Routes> // auth redirect routes exist outside main context - <GoogleAuthRedirectHandlerRoute/> - <GoogleAuthRedirectorRoute/> + <GoogleAuthRedirectHandlerRoute /> + <GoogleAuthRedirectorRoute /> <Route path="" view=BaseRoute> - <Route path="/hot-or-not/:canister_id/:post_id" view=PostView/> - <Route path="/profile/:canister_id/:post_id" view=ProfilePost/> - <Route path="/your-profile/:canister_id/:post_id" view=ProfilePost/> - <Route path="/profile/:id" view=ProfileView/> - <Route path="/your-profile/:id" view=ProfileView/> - <Route path="/upload" view=UploadPostPage/> - <Route path="/error" view=ServerErrorPage/> - <Route path="/menu" view=Menu/> - <Route path="/refer-earn" view=ReferEarn/> - <Route path="/terms-of-service" view=TermsOfService/> - <Route path="/privacy-policy" view=PrivacyPolicy/> - <Route path="/wallet" view=Wallet/> - <Route path="/transactions" view=Transactions/> - <Route path="/leaderboard" view=Leaderboard/> - <Route path="/account-transfer" view=AccountTransfer/> - <Route path="/logout" view=Logout/> - <Route path="" view=RootPage/> + <Route path="/hot-or-not/:canister_id/:post_id" view=PostView /> + <Route path="/profile/:canister_id/:post_id" view=ProfilePost /> + <Route path="/your-profile/:canister_id/:post_id" view=ProfilePost /> + <Route path="/profile/:id" view=ProfileView /> + <Route path="/your-profile/:id" view=ProfileView /> + <Route path="/upload" view=UploadPostPage /> + <Route path="/error" view=ServerErrorPage /> + <Route path="/menu" view=Menu /> + <Route path="/refer-earn" view=ReferEarn /> + <Route path="/terms-of-service" view=TermsOfService /> + <Route path="/privacy-policy" view=PrivacyPolicy /> + <Route path="/wallet" view=Wallet /> + <Route path="/transactions" view=Transactions /> + <Route path="/leaderboard" view=Leaderboard /> + <Route path="/account-transfer" view=AccountTransfer /> + <Route path="/logout" view=Logout /> + <Route path="" view=RootPage /> </Route> </Routes> </main> <nav> - <NavBar/> + <NavBar /> </nav> </Router> } diff --git a/ssr/src/component/scrolling_post_view.rs b/ssr/src/component/scrolling_post_view.rs index 7d3d78a19..e3b6895eb 100644 --- a/ssr/src/component/scrolling_post_view.rs +++ b/ssr/src/component/scrolling_post_view.rs @@ -30,6 +30,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I <For each=move || video_queue().into_iter().enumerate() + // TODO: change it back key=move |(_, details)| (details.canister_id, details.post_id) children=move |(queue_idx, _details)| { let container_ref = create_node_ref::<html::Div>(); @@ -48,7 +49,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I return; } if video_queue.with_untracked(|q| q.len()).saturating_sub(queue_idx) - <= 10 + <= 15 { next_videos.as_ref().map(|nv| { nv() }); } @@ -74,7 +75,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I <div _ref=container_ref class="snap-always snap-end w-full h-full"> <Show when=show_video> <BgView video_queue current_idx idx=queue_idx> - <VideoView video_queue current_idx idx=queue_idx muted/> + <VideoView video_queue current_idx idx=queue_idx muted /> </BgView> </Show> </div> diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 7e7430a0d..085bd110b 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -29,6 +29,7 @@ pub static OFF_CHAIN_AGENT_GRPC_URL: Lazy<Url> = pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); +pub const ML_FEED_GRPC_URL: &str = "https://yral-ml-feed-server-staging-v2.fly.dev:443"; pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/init/mod.rs b/ssr/src/init/mod.rs index b542db464..caad43f47 100644 --- a/ssr/src/init/mod.rs +++ b/ssr/src/init/mod.rs @@ -54,6 +54,7 @@ fn init_google_oauth() -> openidconnect::core::CoreClient { .set_redirect_uri(RedirectUrl::new(redirect_uri).unwrap()) } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] async fn init_grpc_offchain_channel() -> tonic::transport::Channel { use crate::consts::OFF_CHAIN_AGENT_GRPC_URL; @@ -161,6 +162,7 @@ impl AppStateBuilder { cookie_key: init_cookie_key(), #[cfg(feature = "oauth-ssr")] google_oauth: init_google_oauth(), + #[cfg(not(clippy))] #[cfg(feature = "ga4")] grpc_offchain_channel: init_grpc_offchain_channel().await, }; diff --git a/ssr/src/lib.rs b/ssr/src/lib.rs index 7d54b8000..3ea2a6de9 100644 --- a/ssr/src/lib.rs +++ b/ssr/src/lib.rs @@ -14,6 +14,11 @@ pub mod page; pub mod state; pub mod utils; +#[cfg(feature = "hydrate")] +extern crate prost_2 as prost; +#[cfg(feature = "hydrate")] +extern crate tonic_2 as tonic; + #[cfg(feature = "hydrate")] #[wasm_bindgen::prelude::wasm_bindgen] pub fn hydrate() { diff --git a/ssr/src/main.rs b/ssr/src/main.rs index abb535a6d..124591e6a 100644 --- a/ssr/src/main.rs +++ b/ssr/src/main.rs @@ -29,6 +29,8 @@ pub async fn server_fn_handler( provide_context(app_state.cookie_key.clone()); #[cfg(feature = "oauth-ssr")] provide_context(app_state.google_oauth.clone()); + + #[cfg(not(clippy))] #[cfg(feature = "ga4")] provide_context(app_state.grpc_offchain_channel.clone()); }, @@ -54,6 +56,8 @@ pub async fn leptos_routes_handler( provide_context(app_state.cookie_key.clone()); #[cfg(feature = "oauth-ssr")] provide_context(app_state.google_oauth.clone()); + + #[cfg(not(clippy))] #[cfg(feature = "ga4")] provide_context(app_state.grpc_offchain_channel.clone()); }, diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index d234ce7ca..c6da2b0d6 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -23,7 +23,7 @@ use crate::{ route::failure_redirect, }, }; -use video_iter::VideoFetchStream; +use video_iter::{FeedResultType, VideoFetchStream}; use video_loader::{BgView, VideoView}; use overlay::HomeButtonOverlay; @@ -73,7 +73,7 @@ pub fn ScrollingView<NV: Fn() -> NVR + Clone + 'static, NVR>( class="snap-mandatory snap-y overflow-y-scroll h-dvh w-dvw bg-black" style:scroll-snap-points-y="repeat(100vh)" > - <HomeButtonOverlay/> + <HomeButtonOverlay /> <For each=move || video_queue().into_iter().enumerate() key=|(_, details)| (details.canister_id, details.post_id) @@ -123,7 +123,7 @@ pub fn ScrollingView<NV: Fn() -> NVR + Clone + 'static, NVR>( <div _ref=container_ref class="snap-always snap-end w-full h-full"> <Show when=show_video> <BgView video_queue current_idx idx=queue_idx> - <VideoView video_queue current_idx idx=queue_idx muted/> + <VideoView video_queue current_idx idx=queue_idx muted /> </BgView> </Show> </div> @@ -171,7 +171,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { return; } f.start = 1; - f.limit = 1; + f.limit = 15; }); video_queue.update_untracked(|v| { if v.len() > 1 { @@ -198,10 +198,14 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let chunks = if let Some(canisters) = auth_canisters.as_ref() { let fetch_stream = VideoFetchStream::new(canisters, cursor); - fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await + fetch_stream + .fetch_post_uids_hybrid(3, nsfw_enabled, video_queue.get_untracked()) + .await } else { let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); - fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await + fetch_stream + .fetch_post_uids_hybrid(3, nsfw_enabled, video_queue.get_untracked()) + .await }; let res = try_or_redirect!(chunks); @@ -216,14 +220,16 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { } }); } + leptos::logging::log!("feed type: {:?}", res.res_type); + if res.res_type == FeedResultType::PostCache { + fetch_cursor.try_update(|c| c.advance()); + } + if res.end || cnt >= 8 { queue_end.try_set(res.end); break; } - fetch_cursor.try_update(|c| c.advance()); } - - fetch_cursor.try_update(|c| c.advance()); }); create_effect(move |_| { if !recovering_state.get_untracked() { @@ -264,7 +270,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { recovering_state fetch_next_videos=next_videos queue_end - overlay=|| view! { <HomeButtonOverlay/> } + overlay=|| view! { <HomeButtonOverlay /> } /> } } @@ -321,7 +327,7 @@ pub fn PostView() -> impl IntoView { fetch_first_video_uid() .and_then(|initial_post| { let initial_post = initial_post.ok()?; - Some(view! { <PostViewWithUpdates initial_post/> }) + Some(view! { <PostViewWithUpdates initial_post /> }) }) }} diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index f4a768d62..5f6038649 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -23,9 +23,16 @@ pub async fn post_liked_by_me( type PostsStream<'a> = Pin<Box<dyn Stream<Item = Vec<Result<PostDetails, PostViewError>>> + 'a>>; +#[derive(Debug, Eq, PartialEq)] +pub enum FeedResultType { + PostCache, + MLFeed, +} + pub struct FetchVideosRes<'a> { pub posts_stream: PostsStream<'a>, pub end: bool, + pub res_type: FeedResultType, } pub struct VideoFetchStream<'a, const AUTH: bool> { @@ -39,7 +46,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { } pub async fn fetch_post_uids_chunked( - self, + &self, chunks: usize, allow_nsfw: bool, ) -> Result<FetchVideosRes<'a>, PostViewError> { @@ -62,6 +69,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { return Ok(FetchVideosRes { posts_stream: Box::pin(futures::stream::empty()), end: true, + res_type: FeedResultType::PostCache, }) } post_cache::Result_::Err(_) => { @@ -82,6 +90,86 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, + res_type: FeedResultType::PostCache, }) } + + pub async fn fetch_post_uids_ml_feed_chunked( + &self, + chunks: usize, + _allow_nsfw: bool, + video_queue: Vec<PostDetails>, + ) -> Result<FetchVideosRes<'a>, PostViewError> { + #[cfg(feature = "hydrate")] + { + use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; + use leptos::expect_context; + + let user_canister_principal = self.canisters.user_canister(); + let ml_feed: MLFeed = expect_context(); + + let top_posts_fut = ml_feed.get_next_feed( + &user_canister_principal, + self.cursor.limit as u32, + video_queue, + ); + + let top_posts = match top_posts_fut.await { + Ok(top_posts) => top_posts, + Err(e) => { + leptos::logging::log!("error fetching posts: {:?}", e); // TODO: to be removed + return Err(PostViewError::MLFeedError( + "ML feed server failed to send results".into(), + )); + } + }; + + let end = top_posts.len() < self.cursor.limit as usize; + let chunk_stream = top_posts + .into_iter() + .map(move |item| get_post_uid(self.canisters, item.0, item.1)) + .collect::<FuturesOrdered<_>>() + .filter_map(|res| async { res.transpose() }) + .chunks(chunks); + + Ok(FetchVideosRes { + posts_stream: Box::pin(chunk_stream), + end, + res_type: FeedResultType::MLFeed, + }) + } + + #[cfg(not(feature = "hydrate"))] + { + return Ok(FetchVideosRes { + posts_stream: Box::pin(futures::stream::empty()), + end: true, + res_type: FeedResultType::MLFeed, + }); + } + } + + pub async fn fetch_post_uids_hybrid( + &self, + chunks: usize, + _allow_nsfw: bool, + video_queue: Vec<PostDetails>, + ) -> Result<FetchVideosRes<'a>, PostViewError> { + // If cursor.start is < 15, fetch from fetch_post_uids_chunked + // else fetch from fetch_post_uids_ml_feed_chunked + // if that fails fallback to fetch_post_uids_chunked + + if self.cursor.start < 15 { + self.fetch_post_uids_chunked(chunks, _allow_nsfw).await + } else { + let res = self + .fetch_post_uids_ml_feed_chunked(chunks, _allow_nsfw, video_queue) + .await; + + match res { + Ok(res) => Ok(res), + Err(_) => self.fetch_post_uids_chunked(chunks, _allow_nsfw).await, + } + } + } } diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index ff8d437a9..fa49e47f5 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -35,6 +35,31 @@ async fn get_top_post_id() -> Result<Option<(Principal, u64)>, ServerFnError> { Ok(Some((top_item.publisher_canister_id, top_item.post_id))) } +// TODO: Implement this when we shift to the new ml feed for first post +// #[server] +// async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnError> { +// use crate::utils::ml_feed::ml_feed_grpc::get_start_feed; + +// let canisters = unauth_canisters(); +// let user_canister_principal = canisters.user_canister(); +// let top_posts_fut = get_start_feed(&user_canister_principal, 1, vec![]); + +// let top_items = match top_posts_fut.await { +// Ok(top_posts) => top_posts, +// Err(e) => { +// log::error!("failed to fetch top post ml feed: {:?}", e); +// return Err(ServerFnError::ServerError( +// "failed to fetch top post ml feed".to_string(), +// )); +// } +// }; +// let Some(top_item) = top_items.first() else { +// return Ok(None); +// }; + +// Ok(Some((top_item.0, top_item.1))) +// } + #[component] pub fn RootPage() -> impl IntoView { let target_post = create_resource(|| (), |_| get_top_post_id()); @@ -52,7 +77,7 @@ pub fn RootPage() -> impl IntoView { Ok(None) => "/error?err=No Posts Found".to_string(), Err(e) => format!("/error?err={e}"), }; - view! { <Redirect path=url/> } + view! { <Redirect path=url /> } }) }} diff --git a/ssr/src/state/canisters.rs b/ssr/src/state/canisters.rs index 9d7c76bef..b0766073e 100644 --- a/ssr/src/state/canisters.rs +++ b/ssr/src/state/canisters.rs @@ -72,9 +72,9 @@ impl Canisters<true> { .expect("Authenticated canisters must have an identity") } - pub fn user_canister(&self) -> Principal { - self.user_canister - } + // pub fn user_canister(&self) -> Principal { + // self.user_canister + // } pub async fn authenticated_user(&self) -> Result<IndividualUserTemplate<'_>, AgentError> { self.individual_user(self.user_canister).await @@ -94,6 +94,10 @@ impl Canisters<true> { } impl<const A: bool> Canisters<A> { + pub fn user_canister(&self) -> Principal { + self.user_canister + } + pub async fn post_cache(&self) -> Result<PostCache<'_>, AgentError> { let agent = self.agent.get_agent().await?; Ok(PostCache(POST_CACHE_ID, agent)) diff --git a/ssr/src/state/mod.rs b/ssr/src/state/mod.rs index c1b39880e..eb79b2062 100644 --- a/ssr/src/state/mod.rs +++ b/ssr/src/state/mod.rs @@ -8,6 +8,7 @@ pub mod local_storage; #[cfg(feature = "ssr")] pub mod server { + use crate::auth::server_impl::store::KVStoreImpl; use super::canisters::Canisters; @@ -15,7 +16,6 @@ pub mod server { use axum_extra::extract::cookie::Key; use leptos::LeptosOptions; use leptos_router::RouteListing; - use tonic::transport::Channel; #[derive(FromRef, Clone)] pub struct AppState { @@ -30,7 +30,8 @@ pub mod server { pub cookie_key: Key, #[cfg(feature = "oauth-ssr")] pub google_oauth: openidconnect::core::CoreClient, + #[cfg(not(clippy))] #[cfg(feature = "ga4")] - pub grpc_offchain_channel: Channel, + pub grpc_offchain_channel: tonic::transport::Channel, } } diff --git a/ssr/src/utils/event_streaming/mod.rs b/ssr/src/utils/event_streaming/mod.rs index c627c2413..75058b4eb 100644 --- a/ssr/src/utils/event_streaming/mod.rs +++ b/ssr/src/utils/event_streaming/mod.rs @@ -9,6 +9,7 @@ use crate::consts::GTAG_MEASUREMENT_ID; pub mod events; +#[cfg(not(clippy))] #[cfg(feature = "ssr")] pub mod warehouse_events { tonic::include_proto!("warehouse_events"); @@ -64,6 +65,7 @@ pub fn send_event_warehouse(event_name: &str, params: &serde_json::Value) { }); } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] #[server] pub async fn stream_to_offchain_agent(event: String, params: String) -> Result<(), ServerFnError> { @@ -94,3 +96,13 @@ pub async fn stream_to_offchain_agent(event: String, params: String) -> Result<( Ok(()) } + +#[cfg(clippy)] +#[cfg(feature = "ga4")] +#[server] +pub async fn stream_to_offchain_agent( + _event: String, + _params: String, +) -> Result<(), ServerFnError> { + Ok(()) +} diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs new file mode 100644 index 000000000..9affc89a6 --- /dev/null +++ b/ssr/src/utils/ml_feed/mod.rs @@ -0,0 +1,313 @@ +use crate::consts::ML_FEED_GRPC_URL; +use candid::Principal; + +use super::types::PostId; + +#[cfg(feature = "hydrate")] +pub mod ml_feed_grpcweb { + use super::*; + use crate::utils::ml_feed::ml_feed_grpcweb::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, PostItem, + }; + use crate::utils::posts::PostDetails; + use tonic_web_wasm_client::Client; + + pub mod ml_feed_proto { + include!(concat!(env!("OUT_DIR"), "/grpc-web/ml_feed.rs")); + } + + #[derive(Clone)] + pub struct MLFeed { + pub client: MlFeedClient<Client>, + } + + impl Default for MLFeed { + fn default() -> Self { + let client = Client::new(ML_FEED_GRPC_URL.to_string()); + + Self { + client: MlFeedClient::new(client), + } + } + } + + impl MLFeed { + pub async fn get_next_feed( + mut self, + canister_id: &Principal, + limit: u32, + filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic_2::Status> { + let request = FeedRequest { + canister_id: canister_id.to_string(), + filter_posts: filter_list + .iter() + .map(|item| PostItem { + post_id: item.post_id as u32, + canister_id: item.canister_id.to_string(), + video_id: item.uid.clone(), + }) + .collect(), + num_results: limit, + }; + + let response = self.client.get_feed(request).await.map_err(|e| { + tonic_2::Status::new( + tonic_2::Code::Internal, + format!("Error fetching posts: {:?}", e), + ) + })?; + + let feed_res = response.into_inner().feed; + + Ok(feed_res + .iter() + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect()) + } + } +} + +#[cfg(feature = "ssr")] +pub mod ml_feed_grpc { + use super::*; + use crate::utils::posts::PostDetails; + + #[cfg(not(clippy))] + pub mod ml_feed_proto { + tonic::include_proto!("ml_feed"); + } + + #[cfg(not(clippy))] + pub async fn get_start_feed( + canister_id: &Principal, + limit: u32, + filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic::Status> { + use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, PostItem, + }; + use tonic::transport::Channel; + + let channel = Channel::from_static("https://yral-ml-feed-server-staging.fly.dev:443") + .connect() + .await + .expect("Couldn't connect to ML feed server"); + + // let mut client = MlFeedClient::connect(ML_FEED_GRPC_URL).await.unwrap(); + + let mut client = MlFeedClient::new(channel); + + let request = tonic::Request::new(FeedRequest { + canister_id: canister_id.to_string(), + filter_posts: filter_list + .iter() + .map(|item| PostItem { + post_id: item.post_id as u32, + canister_id: item.canister_id.to_string(), + video_id: item.uid.clone(), + }) + .collect(), + num_results: limit, + }); + + let response = client.get_feed(request).await.map_err(|e| { + tonic::Status::new( + tonic::Code::Internal, + format!("error fetching posts: {:?}", e), + ) + })?; + + let feed_res = response.into_inner().feed; + + Ok(feed_res + .iter() + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect()) + } + + // empty function + #[cfg(clippy)] + pub async fn get_start_feed( + _canister_id: &Principal, + _limit: u32, + _filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic::Status> { + Ok(vec![]) + } +} + +// TODO: remove +// #[cfg(feature = "local-feed")] +// pub mod local_feed_impl { +// use super::*; + +// pub async fn get_next_feed() -> Result<Vec<PostId>, tonic_2::Status> { +// let posts = vec![ +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 125, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 124, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 123, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 122, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 121, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 120, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 119, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 118, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 117, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 116, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 115, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 114, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 113, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 112, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 111, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 110, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 109, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 108, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 107, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 106, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 105, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 104, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 103, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 102, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 101, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 100, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 26, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 25, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 24, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 23, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 22, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 21, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 20, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 19, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 18, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 17, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 16, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 15, +// ), +// ]; + +// Ok(posts) +// } +// } diff --git a/ssr/src/utils/mod.rs b/ssr/src/utils/mod.rs index ff0a5d898..a74aac42d 100644 --- a/ssr/src/utils/mod.rs +++ b/ssr/src/utils/mod.rs @@ -4,11 +4,13 @@ use web_time::{Duration, SystemTime}; pub mod event_streaming; pub mod ic; pub mod icon; +pub mod ml_feed; pub mod posts; pub mod profile; pub mod report; pub mod route; pub mod timestamp; +pub mod types; pub mod user; pub mod web; @@ -30,6 +32,7 @@ impl<T> PartialEq for MockPartialEq<T> { } } +#[cfg(not(clippy))] #[cfg(all(feature = "ga4", feature = "ssr"))] pub mod off_chain { tonic::include_proto!("off_chain"); diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 7ffc567c7..cca2125a7 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -5,7 +5,7 @@ use crate::{ canister::individual_user_template::PostDetailsForFrontend, state::canisters::Canisters, }; -use super::profile::propic_from_principal; +use super::{profile::propic_from_principal, types::PostStatus}; use ic_agent::AgentError; use thiserror::Error; @@ -18,6 +18,8 @@ pub enum PostViewError { Canister(String), #[error("http fetch error {0}")] HttpFetch(#[from] reqwest::Error), + #[error("ml feed error {0}")] + MLFeedError(String), } #[derive(Clone, Copy, PartialEq, Debug)] @@ -30,7 +32,7 @@ impl Default for FetchCursor { fn default() -> Self { Self { start: 0, - limit: 10, + limit: 15, } } } @@ -38,7 +40,7 @@ impl Default for FetchCursor { impl FetchCursor { pub fn advance(&mut self) { self.start += self.limit; - self.limit = 25; + self.limit = 30; } } @@ -102,11 +104,21 @@ pub async fn get_post_uid<const AUTH: bool>( { Ok(p) => p, Err(e) => { - log::warn!("failed to get post details: {}, skipping", e); + log::warn!( + "failed to get post details for {} {}: {}, skipping", + user_canister.to_string(), + post_id, + e + ); return Ok(None); } }; + // TODO: temporary patch in frontend to not show banned videos, to be removed later after NSFW tagging + if PostStatus::from(&post_details.status) == PostStatus::BannedDueToUserReporting { + return Ok(None); + } + let post_uuid = &post_details.video_uid; let req_url = format!( "https://customer-2p3jflss4r4hmpnz.cloudflarestream.com/{}/manifest/video.m3u8", diff --git a/ssr/src/utils/report.rs b/ssr/src/utils/report.rs index 79d35296b..36f55ac57 100644 --- a/ssr/src/utils/report.rs +++ b/ssr/src/utils/report.rs @@ -1,6 +1,6 @@ use std::{env, fmt::Display}; -use leptos::{expect_context, server, ServerFnError}; +use leptos::{server, ServerFnError}; pub enum ReportOption { Nudity, @@ -22,6 +22,7 @@ impl ReportOption { } } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] #[server] pub async fn send_report_offchain( @@ -34,6 +35,7 @@ pub async fn send_report_offchain( video_url: String, ) -> Result<(), ServerFnError> { use crate::utils::off_chain; + use leptos::expect_context; use tonic::metadata::MetadataValue; use tonic::transport::Channel; use tonic::Request; @@ -68,3 +70,18 @@ pub async fn send_report_offchain( Ok(()) } + +#[cfg(clippy)] +#[cfg(feature = "ga4")] +#[server] +pub async fn send_report_offchain( + _reporter_id: String, + _publisher_id: String, + _publisher_canister_id: String, + _post_id: String, + _video_id: String, + _reason: String, + _video_url: String, +) -> Result<(), ServerFnError> { + Ok(()) +} diff --git a/ssr/src/utils/types.rs b/ssr/src/utils/types.rs new file mode 100644 index 000000000..5d4a7fe4b --- /dev/null +++ b/ssr/src/utils/types.rs @@ -0,0 +1,29 @@ +use crate::canister::individual_user_template::PostStatus as PostStatusCandid; +use candid::Principal; + +pub type PostId = (Principal, u64); + +#[derive(PartialEq, Debug, Eq)] +pub enum PostStatus { + BannedForExplicitness, + BannedDueToUserReporting, + Uploaded, + CheckingExplicitness, + ReadyToView, + Transcoding, + Deleted, +} + +impl From<&PostStatusCandid> for PostStatus { + fn from(status: &PostStatusCandid) -> Self { + match status { + PostStatusCandid::BannedForExplicitness => PostStatus::BannedForExplicitness, + PostStatusCandid::BannedDueToUserReporting => PostStatus::BannedDueToUserReporting, + PostStatusCandid::Uploaded => PostStatus::Uploaded, + PostStatusCandid::CheckingExplicitness => PostStatus::CheckingExplicitness, + PostStatusCandid::ReadyToView => PostStatus::ReadyToView, + PostStatusCandid::Transcoding => PostStatus::Transcoding, + PostStatusCandid::Deleted => PostStatus::Deleted, + } + } +}