From 2378e5f0b3fcb8ba62774e35ba14b3fac2c6281d Mon Sep 17 00:00:00 2001 From: komal-rs Date: Fri, 2 Aug 2024 15:30:31 +0530 Subject: [PATCH 1/9] mlfeed --- ssr/Cargo.toml | 19 ++--- ssr/build.rs | 13 ++++ ssr/contracts | 2 +- ssr/src/app.rs | 67 +++++++++-------- ssr/src/consts/mod.rs | 2 + ssr/src/init/mod.rs | 2 + ssr/src/page/post_view/mod.rs | 17 +++-- ssr/src/page/post_view/video_iter.rs | 54 +++++++++++++- ssr/src/state/mod.rs | 2 +- ssr/src/utils/ml_feed/mod.rs | 105 +++++++++++++++++++++++++++ ssr/src/utils/mod.rs | 2 + ssr/src/utils/posts.rs | 7 +- ssr/src/utils/types.rs | 1 + 13 files changed, 240 insertions(+), 53 deletions(-) create mode 100644 ssr/src/utils/ml_feed/mod.rs create mode 100644 ssr/src/utils/types.rs diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 37731458..4552ce06 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -16,7 +16,10 @@ leptos_meta = { version = "0.6", features = ["nightly"] } leptos_router = { version = "0.6", features = ["nightly"] } log = "0.4" simple_logger = "4.0" -tokio = { version = "1", optional = true, features = ["rt-multi-thread", "signal"] } +tokio = { version = "1", optional = true, features = [ + "rt-multi-thread", + "signal", +] } tower = { version = "0.4", optional = true } tower-http = { version = "0.5", features = ["fs"], optional = true } wasm-bindgen = "=0.2.92" @@ -110,7 +113,8 @@ hydrate = [ "dep:web-sys", "reqwest/native-tls", "dep:rand_chacha", - "dep:wasm-bindgen-futures" + "dep:wasm-bindgen-futures", + "tonic", ] ssr = [ "dep:axum", @@ -150,6 +154,7 @@ redis-kv = [] cloudflare = ["dep:gob-cloudflare"] backend-admin = [] ga4 = [] +local-feed = [] mock-wallet-history = ["dep:rand_chacha"] release-bin = [ "ssr", @@ -175,13 +180,9 @@ local-bin = [ "backend-admin", "dep:testcontainers", "dep:yral-testcontainers", + "local-feed", ] -local-lib = [ - "hydrate", - "redis-kv", - "local-auth", - "backend-admin", -] +local-lib = ["hydrate", "redis-kv", "local-auth", "backend-admin", "local-feed"] [package.metadata.leptos] # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name @@ -238,7 +239,7 @@ env = "DEV" # The features to use when compiling the bin target # # Optional. Can be over-ridden with the command line parameter --bin-features -bin-features = ["ssr","local-auth"] +bin-features = ["ssr", "local-auth"] # If the --no-default-features flag should be used when compiling the bin target # diff --git a/ssr/build.rs b/ssr/build.rs index c631bd82..e5115cb9 100644 --- a/ssr/build.rs +++ b/ssr/build.rs @@ -126,8 +126,21 @@ mod build_common { Ok(()) } + fn build_gprc_client() -> Result<()> { + let ml_feed_proto = "contracts/projects/ml_feed/ml_feed.proto"; + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + tonic_build::configure() + .build_client(true) + .build_server(false) + .out_dir(out_dir) + .compile(&[ml_feed_proto], &["proto"])?; + Ok(()) + } + pub fn build_common() -> Result<()> { build_did_intf()?; + build_gprc_client()?; Ok(()) } } diff --git a/ssr/contracts b/ssr/contracts index dd72cc15..034e7001 160000 --- a/ssr/contracts +++ b/ssr/contracts @@ -1 +1 @@ -Subproject commit dd72cc15f9d87d7d5b9e51bfe2c77f06971f98a4 +Subproject commit 034e70016d8163e561b0e8ff5be0e9be1c990d20 diff --git a/ssr/src/app.rs b/ssr/src/app.rs index 6c38fddb..170bc9b9 100644 --- a/ssr/src/app.rs +++ b/ssr/src/app.rs @@ -16,7 +16,10 @@ use crate::{ wallet::{transactions::Transactions, Wallet}, }, state::{canisters::Canisters, content_seed_client::ContentSeedClient, history::HistoryCtx}, - utils::event_streaming::EventHistory, + utils::{ + event_streaming::EventHistory, + ml_feed::{self, MLFeed}, + }, }; use leptos::*; use leptos_meta::*; @@ -26,7 +29,7 @@ use leptos_router::*; fn NotFound() -> impl IntoView { let mut outside_errors = Errors::default(); outside_errors.insert_with_default_key(AppError::NotFound); - view! { } + view! { } } #[component(transparent)] @@ -35,11 +38,11 @@ fn GoogleAuthRedirectHandlerRoute() -> impl IntoView { #[cfg(any(feature = "oauth-ssr", feature = "oauth-hydrate"))] { use crate::page::google_redirect::GoogleRedirectHandler; - view! { } + view! { } } #[cfg(not(any(feature = "oauth-ssr", feature = "oauth-hydrate")))] { - view! { } + view! { } } } @@ -49,11 +52,11 @@ fn GoogleAuthRedirectorRoute() -> impl IntoView { #[cfg(any(feature = "oauth-ssr", feature = "oauth-hydrate"))] { use crate::page::google_redirect::GoogleRedirector; - view! { } + view! { } } #[cfg(not(any(feature = "oauth-ssr", feature = "oauth-hydrate")))] { - view! { } + view! { } } } @@ -67,6 +70,10 @@ pub fn App() -> impl IntoView { provide_context(ProfilePostsContext::default()); provide_context(AuthorizedUserToSeedContent::default()); + // ML Feed + let ml_feed = create_rw_signal(MLFeed::default()); + provide_context(ml_feed); + // History Tracking let history_ctx = HistoryCtx::default(); provide_context(history_ctx.clone()); @@ -85,12 +92,12 @@ pub fn App() -> impl IntoView { } view! { - + // sets the document title - + <Title text="Yral" /> - <Link rel="manifest" href="/app.webmanifest"/> + <Link rel="manifest" href="/app.webmanifest" /> // GA4 Global Site Tag (gtag.js) - Google Analytics // G-6W5Q2MRX0E to test locally | G-PLNNETMSLM @@ -110,36 +117,36 @@ pub fn App() -> impl IntoView { </Show> // content for this welcome page - <Router fallback=|| view! { <NotFound/> }.into_view()> + <Router fallback=|| view! { <NotFound /> }.into_view()> <main> <Routes> // auth redirect routes exist outside main context - <GoogleAuthRedirectHandlerRoute/> - <GoogleAuthRedirectorRoute/> + <GoogleAuthRedirectHandlerRoute /> + <GoogleAuthRedirectorRoute /> <Route path="" view=BaseRoute> - <Route path="/hot-or-not/:canister_id/:post_id" view=PostView/> - <Route path="/profile/:canister_id/:post_id" view=ProfilePost/> - <Route path="/your-profile/:canister_id/:post_id" view=ProfilePost/> - <Route path="/profile/:id" view=ProfileView/> - <Route path="/your-profile/:id" view=ProfileView/> - <Route path="/upload" view=UploadPostPage/> - <Route path="/error" view=ServerErrorPage/> - <Route path="/menu" view=Menu/> - <Route path="/refer-earn" view=ReferEarn/> - <Route path="/terms-of-service" view=TermsOfService/> - <Route path="/privacy-policy" view=PrivacyPolicy/> - <Route path="/wallet" view=Wallet/> - <Route path="/transactions" view=Transactions/> - <Route path="/leaderboard" view=Leaderboard/> - <Route path="/account-transfer" view=AccountTransfer/> - <Route path="/logout" view=Logout/> - <Route path="" view=RootPage/> + <Route path="/hot-or-not/:canister_id/:post_id" view=PostView /> + <Route path="/profile/:canister_id/:post_id" view=ProfilePost /> + <Route path="/your-profile/:canister_id/:post_id" view=ProfilePost /> + <Route path="/profile/:id" view=ProfileView /> + <Route path="/your-profile/:id" view=ProfileView /> + <Route path="/upload" view=UploadPostPage /> + <Route path="/error" view=ServerErrorPage /> + <Route path="/menu" view=Menu /> + <Route path="/refer-earn" view=ReferEarn /> + <Route path="/terms-of-service" view=TermsOfService /> + <Route path="/privacy-policy" view=PrivacyPolicy /> + <Route path="/wallet" view=Wallet /> + <Route path="/transactions" view=Transactions /> + <Route path="/leaderboard" view=Leaderboard /> + <Route path="/account-transfer" view=AccountTransfer /> + <Route path="/logout" view=Logout /> + <Route path="" view=RootPage /> </Route> </Routes> </main> <nav> - <NavBar/> + <NavBar /> </nav> </Router> } diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 1dac08f2..589c6ab3 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -26,6 +26,8 @@ pub static OFF_CHAIN_AGENT_GRPC_URL: Lazy<Url> = pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); +pub static ML_FEED_GRPC_URL: Lazy<Url> = + Lazy::new(|| Url::parse("https://yral-ml-feed.fly.dev:443").unwrap()); pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/init/mod.rs b/ssr/src/init/mod.rs index b542db46..c01c6a26 100644 --- a/ssr/src/init/mod.rs +++ b/ssr/src/init/mod.rs @@ -66,6 +66,8 @@ async fn init_grpc_offchain_channel() -> tonic::transport::Channel { .expect("Couldn't connect to off-chain agent") } + + #[cfg(feature = "backend-admin")] fn init_admin_canisters() -> crate::state::admin_canisters::AdminCanisters { use crate::state::admin_canisters::AdminCanisters; diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index d234ce7c..192065c7 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -19,8 +19,7 @@ use crate::{ state::canisters::{unauth_canisters, Canisters}, try_or_redirect, utils::{ - posts::{get_post_uid, FetchCursor, PostDetails}, - route::failure_redirect, + ml_feed::MLFeed, posts::{get_post_uid, FetchCursor, PostDetails}, route::failure_redirect }, }; use video_iter::VideoFetchStream; @@ -73,7 +72,7 @@ pub fn ScrollingView<NV: Fn() -> NVR + Clone + 'static, NVR>( class="snap-mandatory snap-y overflow-y-scroll h-dvh w-dvw bg-black" style:scroll-snap-points-y="repeat(100vh)" > - <HomeButtonOverlay/> + <HomeButtonOverlay /> <For each=move || video_queue().into_iter().enumerate() key=|(_, details)| (details.canister_id, details.post_id) @@ -123,7 +122,7 @@ pub fn ScrollingView<NV: Fn() -> NVR + Clone + 'static, NVR>( <div _ref=container_ref class="snap-always snap-end w-full h-full"> <Show when=show_video> <BgView video_queue current_idx idx=queue_idx> - <VideoView video_queue current_idx idx=queue_idx muted/> + <VideoView video_queue current_idx idx=queue_idx muted /> </BgView> </Show> </div> @@ -186,6 +185,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { } let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); + let ml_feed: RwSignal<MLFeed> = expect_context(); let fetch_video_action = create_action(move |_| async move { loop { @@ -195,12 +195,13 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let auth_canisters = auth_canisters.get_untracked(); let nsfw_enabled = nsfw_enabled.get_untracked(); let unauth_canisters = unauth_canisters(); + let ml_feed = ml_feed.get_untracked(); let chunks = if let Some(canisters) = auth_canisters.as_ref() { - let fetch_stream = VideoFetchStream::new(canisters, cursor); + let fetch_stream = VideoFetchStream::new(canisters, &ml_feed, cursor); fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await } else { - let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); + let fetch_stream = VideoFetchStream::new(&unauth_canisters, &ml_feed, cursor); fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await }; @@ -264,7 +265,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { recovering_state fetch_next_videos=next_videos queue_end - overlay=|| view! { <HomeButtonOverlay/> } + overlay=|| view! { <HomeButtonOverlay /> } /> } } @@ -321,7 +322,7 @@ pub fn PostView() -> impl IntoView { fetch_first_video_uid() .and_then(|initial_post| { let initial_post = initial_post.ok()?; - Some(view! { <PostViewWithUpdates initial_post/> }) + Some(view! { <PostViewWithUpdates initial_post /> }) }) }} diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index f4a768d6..a1e198b5 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -6,7 +6,7 @@ use futures::{stream::FuturesOrdered, Stream, StreamExt}; use crate::{ canister::post_cache::{self, NsfwFilter}, state::canisters::Canisters, - utils::posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}, + utils::{ml_feed::{MLFeed}, posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}}, }; pub async fn post_liked_by_me( @@ -30,12 +30,13 @@ pub struct FetchVideosRes<'a> { pub struct VideoFetchStream<'a, const AUTH: bool> { canisters: &'a Canisters<AUTH>, + ml_feed: &'a MLFeed, cursor: FetchCursor, } impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { - pub fn new(canisters: &'a Canisters<AUTH>, cursor: FetchCursor) -> Self { - Self { canisters, cursor } + pub fn new(canisters: &'a Canisters<AUTH>, ml_feed: &'a MLFeed, cursor: FetchCursor) -> Self { + Self { canisters, ml_feed, cursor } } pub async fn fetch_post_uids_chunked( @@ -84,4 +85,51 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { end, }) } + + pub async fn fetch_post_uids_chunked_mlfeed( + self, + chunks: usize, + allow_nsfw: bool, + ) -> Result<FetchVideosRes<'a>, PostViewError> { + + let mut posts_fut; + + #[cfg(not(feature = "local-feed"))] + { + use crate::utils::ml_feed::ml_feed_impl; + + posts_fut = ml_feed_impl::get_next_feed(); + } + + #[cfg(feature = "local-feed")] + { + use crate::utils::ml_feed::local_feed_impl; + + posts_fut = local_feed_impl::get_next_feed(); + } + + + let posts = match posts_fut.await? { + Ok(posts) => posts, + Err(_) => { + // TODO: change this error type + return Err(PostViewError::Canister( + "Feed server error".into(), + )) + } + }; + + let end = posts.len() < self.cursor.limit as usize; + let chunk_stream = posts + .into_iter() + .map(move |item| get_post_uid(self.canisters, item.publisher_canister_id, item.post_id)) + .collect::<FuturesOrdered<_>>() + .filter_map(|res| async { res.transpose() }) + .chunks(chunks); + + Ok(FetchVideosRes { + posts_stream: Box::pin(chunk_stream), + end, + }) + } } diff --git a/ssr/src/state/mod.rs b/ssr/src/state/mod.rs index c1b39880..83dfd93f 100644 --- a/ssr/src/state/mod.rs +++ b/ssr/src/state/mod.rs @@ -8,7 +8,7 @@ pub mod local_storage; #[cfg(feature = "ssr")] pub mod server { - use crate::auth::server_impl::store::KVStoreImpl; + use crate::{auth::server_impl::store::KVStoreImpl, utils::ml_feed::MLFeedGRPCChannel}; use super::canisters::Canisters; use axum::extract::FromRef; diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs new file mode 100644 index 00000000..bca1a3c4 --- /dev/null +++ b/ssr/src/utils/ml_feed/mod.rs @@ -0,0 +1,105 @@ +use candid::Principal; +use leptos::RwSignal; +use tonic::transport::Channel; +use crate::consts::ML_FEED_GRPC_URL; + +use super::types::PostId; + +#[derive(Clone, Default)] +pub struct MLFeed { + pub channel: Option<Channel>, +} + + +pub async fn init_mlfeed_grpc_channel() -> Channel { + let ml_feed_url = ML_FEED_GRPC_URL.as_ref(); + Channel::from_static(ml_feed_url) + .connect() + .await + .expect("Couldn't connect to ML feed server") +} + +impl MLFeed { + pub async fn get_channel(&mut self) -> &Channel { + if self.channel.is_none() { + self.channel = Some(init_mlfeed_grpc_channel().await); + } + self.channel.as_ref().unwrap() + } +} + + +#[cfg(not(feature = "local-feed"))] +pub mod ml_feed_impl { + use super::*; + + pub async fn get_next_feed( + ml_feed_channel: MLFeed, + canister_id: &Principal, + limit: u32, + filter_list: Vec<PostId>, + ) -> Result<Vec<PostId>, tonic::Status> { + + // let mut client = MLFeedServiceClient::new(self.channel.clone()); + // let request = tonic::Request::new(MLFeedRequest { + // user_id: user_id.to_string(), + // limit: limit as u32, + // }); + // let response = client.get_recommendations(request).await?; + // Ok(response.into_inner().recommendations) + + Ok(vec![]) + } +} + +#[cfg(feature = "local-feed")] +pub mod local_feed_impl { + use super::*; + + pub async fn get_next_feed() -> Result<Vec<PostId>, tonic::Status> { + + let posts = vec![ + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 125), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 124), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 123), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 122), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 121), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 120), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 119), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 118), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 117), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 116), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 115), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 114), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 113), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 112), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 111), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 110), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 109), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 108), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 107), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 106), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 105), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 104), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 103), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 102), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 101), + (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 100), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 26), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 25), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 24), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 23), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 22), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 21), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 20), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 19), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 18), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 17), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 16), + (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 15), + ]; + + Ok(posts) + } +} + diff --git a/ssr/src/utils/mod.rs b/ssr/src/utils/mod.rs index ff0a5d89..05ebb55f 100644 --- a/ssr/src/utils/mod.rs +++ b/ssr/src/utils/mod.rs @@ -4,11 +4,13 @@ use web_time::{Duration, SystemTime}; pub mod event_streaming; pub mod ic; pub mod icon; +pub mod ml_feed; pub mod posts; pub mod profile; pub mod report; pub mod route; pub mod timestamp; +pub mod types; pub mod user; pub mod web; diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 7ffc567c..0f4cf19d 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -2,7 +2,7 @@ use candid::Principal; use serde::{Deserialize, Serialize}; use crate::{ - canister::individual_user_template::PostDetailsForFrontend, state::canisters::Canisters, + canister::individual_user_template::{PostDetailsForFrontend, PostStatus}, state::canisters::Canisters, }; use super::profile::propic_from_principal; @@ -107,6 +107,11 @@ pub async fn get_post_uid<const AUTH: bool>( } }; + // TODO: Add this filter + // if post_details.status == PostStatus::BannedDueToUserReporting { + // return Ok(None); + // } + let post_uuid = &post_details.video_uid; let req_url = format!( "https://customer-2p3jflss4r4hmpnz.cloudflarestream.com/{}/manifest/video.m3u8", diff --git a/ssr/src/utils/types.rs b/ssr/src/utils/types.rs new file mode 100644 index 00000000..d58c6676 --- /dev/null +++ b/ssr/src/utils/types.rs @@ -0,0 +1 @@ +pub type PostId = (Principal, u64); From 5691bf03bd0ad543c7e53642b890401c4d3a80d3 Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Mon, 5 Aug 2024 23:52:38 +0530 Subject: [PATCH 2/9] ml feed testing --- Cargo.lock | 208 ++++++++++++-------- ssr/Cargo.toml | 24 ++- ssr/build.rs | 4 +- ssr/contracts | 2 +- ssr/src/app.rs | 13 +- ssr/src/consts/mod.rs | 2 +- ssr/src/lib.rs | 3 + ssr/src/page/post_view/mod.rs | 36 +++- ssr/src/page/post_view/video_iter.rs | 125 +++++++----- ssr/src/state/canisters.rs | 10 +- ssr/src/state/mod.rs | 2 +- ssr/src/utils/ml_feed/mod.rs | 275 +++++++++++++++++++-------- ssr/src/utils/mod.rs | 122 +++++++++++- ssr/src/utils/posts.rs | 4 +- ssr/src/utils/types.rs | 2 + 15 files changed, 595 insertions(+), 237 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e65a2197..2d9f2c40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,34 +225,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 0.1.2", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.7.5" @@ -260,7 +232,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core 0.4.3", + "axum-core", "axum-macros", "bytes", "futures-util", @@ -289,23 +261,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.4.3" @@ -333,8 +288,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum 0.7.5", - "axum-core 0.4.3", + "axum", + "axum-core", "bytes", "cookie", "futures-util", @@ -2578,7 +2533,7 @@ name = "hot-or-not-web-leptos-ssr" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.5", + "axum", "axum-extra", "bb8", "bb8-redis", @@ -2611,7 +2566,7 @@ dependencies = [ "log", "once_cell", "openidconnect", - "prost", + "prost 0.13.1", "rand_chacha", "redb", "redis", @@ -2625,7 +2580,9 @@ dependencies = [ "thiserror", "tokio", "tonic", - "tonic-build", + "tonic-build 0.11.0", + "tonic-build 0.12.1", + "tonic-web-wasm-client", "tower", "tower-http", "tracing", @@ -2836,14 +2793,15 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ - "hyper 0.14.30", + "hyper 1.4.1", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service", ] [[package]] @@ -3470,7 +3428,7 @@ version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3923af454949eb7a5ea9a89d5fdc4d21e4850eb8d47d942d35355e5079444867" dependencies = [ - "axum 0.7.5", + "axum", "cfg-if", "futures", "http-body-util", @@ -4529,7 +4487,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +dependencies = [ + "bytes", + "prost-derive 0.13.1", ] [[package]] @@ -4546,8 +4514,29 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn 2.0.72", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.13.1", + "prost-types 0.13.1", "regex", "syn 2.0.72", "tempfile", @@ -4566,13 +4555,35 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "prost-derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", +] + +[[package]] +name = "prost-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +dependencies = [ + "prost 0.13.1", ] [[package]] @@ -5035,6 +5046,7 @@ version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -5406,7 +5418,7 @@ version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9aaae169927ef701a4734d680adcb08f13269c9f0af822bf175d681fe56d65c" dependencies = [ - "axum 0.7.5", + "axum", "bytes", "ciborium", "const_format", @@ -5886,16 +5898,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.4.0" @@ -6033,27 +6035,29 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", - "base64 0.21.7", + "axum", + "base64 0.22.1", "bytes", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.1", "rustls-pemfile 2.1.2", - "rustls-pki-types", + "socket2", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-stream", "tower", "tower-layer", @@ -6070,11 +6074,49 @@ checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", + "prost-build 0.12.6", "quote", "syn 2.0.72", ] +[[package]] +name = "tonic-build" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568392c5a2bd0020723e3f387891176aabafe36fd9fcd074ad309dfa0c8eb964" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.13.1", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tonic-web-wasm-client" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5ca6e7bdd0042c440d36b6df97c1436f1d45871ce18298091f114004b1beb4" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "httparse", + "js-sys", + "pin-project", + "thiserror", + "tonic", + "tower-service", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 4552ce06..74a0e6e3 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -86,15 +86,23 @@ gob-cloudflare = { git = "https://github.com/go-bazzinga/gob-cloudflare", rev = yral-metadata-client = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } yral-metadata-types = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } gloo-utils = { version = "0.2.0", features = ["serde"] } -tonic = { version = "0.11.0", features = [ +tonic = { version = "0.12.1", features = [ "tls", "tls-webpki-roots", ], optional = true } -prost = { version = "0.12.4", optional = true } +prost = { version = "0.13.1", optional = true } hmac = { version = "0.12.1", optional = true } wasm-bindgen-futures = { version = "0.4.42", optional = true } testcontainers = { version = "0.20.0", optional = true } yral-testcontainers = { git = "https://github.com/go-bazzinga/yral-testcontainers", rev = "f9d2c01c498d58fca0595a48bdc3f9400e57ec2f", optional = true } +tonic-web-wasm-client = { version = "0.6" } + +[dependencies.tonic_2] +package = "tonic" +version = "0.12.1" +optional = true +default-features = false +features = ["prost", "codegen"] [build-dependencies] serde = { version = "1.0", features = ["derive"] } @@ -104,6 +112,12 @@ convert_case = "0.6.0" tonic-build = "0.11.0" anyhow = "1.0.86" +[build-dependencies.tonic_build_2] +package = "tonic-build" +version = "0.12" +default-features = false +features = ["prost"] + [features] hydrate = [ "leptos/hydrate", @@ -114,7 +128,8 @@ hydrate = [ "reqwest/native-tls", "dep:rand_chacha", "dep:wasm-bindgen-futures", - "tonic", + "tonic_2", + "prost", ] ssr = [ "dep:axum", @@ -180,9 +195,8 @@ local-bin = [ "backend-admin", "dep:testcontainers", "dep:yral-testcontainers", - "local-feed", ] -local-lib = ["hydrate", "redis-kv", "local-auth", "backend-admin", "local-feed"] +local-lib = ["hydrate", "redis-kv", "local-auth", "backend-admin"] [package.metadata.leptos] # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name diff --git a/ssr/build.rs b/ssr/build.rs index e5115cb9..60d48143 100644 --- a/ssr/build.rs +++ b/ssr/build.rs @@ -130,7 +130,7 @@ mod build_common { let ml_feed_proto = "contracts/projects/ml_feed/ml_feed.proto"; let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - tonic_build::configure() + tonic_build_2::configure() .build_client(true) .build_server(false) .out_dir(out_dir) @@ -140,7 +140,9 @@ mod build_common { pub fn build_common() -> Result<()> { build_did_intf()?; + build_gprc_client()?; + Ok(()) } } diff --git a/ssr/contracts b/ssr/contracts index 034e7001..50066890 160000 --- a/ssr/contracts +++ b/ssr/contracts @@ -1 +1 @@ -Subproject commit 034e70016d8163e561b0e8ff5be0e9be1c990d20 +Subproject commit 50066890f6f8851538a2ccecd03cbe3ffd236c2c diff --git a/ssr/src/app.rs b/ssr/src/app.rs index 170bc9b9..c72caca6 100644 --- a/ssr/src/app.rs +++ b/ssr/src/app.rs @@ -16,10 +16,7 @@ use crate::{ wallet::{transactions::Transactions, Wallet}, }, state::{canisters::Canisters, content_seed_client::ContentSeedClient, history::HistoryCtx}, - utils::{ - event_streaming::EventHistory, - ml_feed::{self, MLFeed}, - }, + utils::event_streaming::EventHistory, }; use leptos::*; use leptos_meta::*; @@ -70,9 +67,11 @@ pub fn App() -> impl IntoView { provide_context(ProfilePostsContext::default()); provide_context(AuthorizedUserToSeedContent::default()); - // ML Feed - let ml_feed = create_rw_signal(MLFeed::default()); - provide_context(ml_feed); + #[cfg(feature = "hydrate")] + { + use crate::utils::ml_feed::MLFeed; + provide_context(MLFeed::default()); + } // History Tracking let history_ctx = HistoryCtx::default(); diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 589c6ab3..39046978 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -27,7 +27,7 @@ pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); pub static ML_FEED_GRPC_URL: Lazy<Url> = - Lazy::new(|| Url::parse("https://yral-ml-feed.fly.dev:443").unwrap()); + Lazy::new(|| Url::parse("http://127.0.0.1:50051").unwrap()); pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/lib.rs b/ssr/src/lib.rs index 7d54b800..0ca490a6 100644 --- a/ssr/src/lib.rs +++ b/ssr/src/lib.rs @@ -14,6 +14,9 @@ pub mod page; pub mod state; pub mod utils; +#[cfg(feature = "hydrate")] +extern crate tonic_2 as tonic; + #[cfg(feature = "hydrate")] #[wasm_bindgen::prelude::wasm_bindgen] pub fn hydrate() { diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index 192065c7..9e96df37 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -19,7 +19,7 @@ use crate::{ state::canisters::{unauth_canisters, Canisters}, try_or_redirect, utils::{ - ml_feed::MLFeed, posts::{get_post_uid, FetchCursor, PostDetails}, route::failure_redirect + posts::{get_post_uid, FetchCursor, PostDetails}, route::failure_redirect }, }; use video_iter::VideoFetchStream; @@ -185,9 +185,22 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { } let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); - let ml_feed: RwSignal<MLFeed> = expect_context(); + // let ml_feed: MLFeed = expect_context(); - let fetch_video_action = create_action(move |_| async move { + // #[cfg(feature = "hydrate")] + // { + // use crate::utils::ml_feed::ml_feed_impl::get_next_feed; + + // leptos::spawn_local(async move { + + // let temp_res = get_next_feed(&Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 10, vec![]).await; + + // logging::log!("temp_res: {:?}", temp_res); + // }); + + // } + + let fetch_video_action = create_action(move |_|{ async move { loop { let Some(cursor) = fetch_cursor.try_get_untracked() else { return; @@ -195,14 +208,16 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let auth_canisters = auth_canisters.get_untracked(); let nsfw_enabled = nsfw_enabled.get_untracked(); let unauth_canisters = unauth_canisters(); - let ml_feed = ml_feed.get_untracked(); let chunks = if let Some(canisters) = auth_canisters.as_ref() { - let fetch_stream = VideoFetchStream::new(canisters, &ml_feed, cursor); - fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await + leptos::logging::log!("auth_canisters: yo"); + let fetch_stream = VideoFetchStream::new(canisters, cursor); + fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled).await // fetch_post_uids_ml_feed_chunked } else { - let fetch_stream = VideoFetchStream::new(&unauth_canisters, &ml_feed, cursor); - fetch_stream.fetch_post_uids_chunked(3, nsfw_enabled).await + leptos::logging::log!("unauth_canisters: yo"); + // return; + let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); + fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled).await // fetch_post_uids_chunked }; let res = try_or_redirect!(chunks); @@ -214,18 +229,21 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { for uid in chunk { let uid = try_or_redirect!(uid); q.push(uid); + + leptos::logging::log!("queue len: {:?}, cur_idc: {:?}", q.len(), current_idx.get_untracked()); } }); } if res.end || cnt >= 8 { queue_end.try_set(res.end); + leptos::logging::log!("breaking: queue_end {:?}", queue_end.get_untracked()); break; } fetch_cursor.try_update(|c| c.advance()); } fetch_cursor.try_update(|c| c.advance()); - }); + }}); create_effect(move |_| { if !recovering_state.get_untracked() { fetch_video_action.dispatch(()); diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index a1e198b5..27a3a004 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -6,7 +6,7 @@ use futures::{stream::FuturesOrdered, Stream, StreamExt}; use crate::{ canister::post_cache::{self, NsfwFilter}, state::canisters::Canisters, - utils::{ml_feed::{MLFeed}, posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}}, + utils::{posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}}, }; pub async fn post_liked_by_me( @@ -30,13 +30,12 @@ pub struct FetchVideosRes<'a> { pub struct VideoFetchStream<'a, const AUTH: bool> { canisters: &'a Canisters<AUTH>, - ml_feed: &'a MLFeed, cursor: FetchCursor, } impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { - pub fn new(canisters: &'a Canisters<AUTH>, ml_feed: &'a MLFeed, cursor: FetchCursor) -> Self { - Self { canisters, ml_feed, cursor } + pub fn new(canisters: &'a Canisters<AUTH>, cursor: FetchCursor) -> Self { + Self { canisters, cursor } } pub async fn fetch_post_uids_chunked( @@ -86,50 +85,88 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { }) } - pub async fn fetch_post_uids_chunked_mlfeed( + pub async fn fetch_post_uids_ml_feed_chunked( self, chunks: usize, allow_nsfw: bool, ) -> Result<FetchVideosRes<'a>, PostViewError> { - let mut posts_fut; - - #[cfg(not(feature = "local-feed"))] - { - use crate::utils::ml_feed::ml_feed_impl; - - posts_fut = ml_feed_impl::get_next_feed(); - } - - #[cfg(feature = "local-feed")] - { - use crate::utils::ml_feed::local_feed_impl; - - posts_fut = local_feed_impl::get_next_feed(); - } - - - let posts = match posts_fut.await? { - Ok(posts) => posts, - Err(_) => { - // TODO: change this error type - return Err(PostViewError::Canister( - "Feed server error".into(), - )) - } - }; - - let end = posts.len() < self.cursor.limit as usize; - let chunk_stream = posts - .into_iter() - .map(move |item| get_post_uid(self.canisters, item.publisher_canister_id, item.post_id)) - .collect::<FuturesOrdered<_>>() - .filter_map(|res| async { res.transpose() }) - .chunks(chunks); - - Ok(FetchVideosRes { - posts_stream: Box::pin(chunk_stream), - end, - }) + // #[cfg(feature = "hydrate")] + // { + // leptos::logging::log!("in hydrate"); + + // use crate::utils::ml_feed::ml_feed_impl::get_next_feed; + + // let user_canister_principal = self.canisters.user_canister(); + + // let top_posts_fut = get_next_feed(&user_canister_principal, self.cursor.limit as u32, vec![]); + + // let top_posts = match top_posts_fut.await { + // Ok(top_posts) => top_posts, + // Err(e) => { + // leptos::logging::log!("error fetching posts: {:?}", e); + // return Ok(FetchVideosRes { + // posts_stream: Box::pin(futures::stream::empty()), + // end: true, + // }) + // } + // }; + // leptos::logging::log!("in hydrate - after first ret : top_posts : {:?}", top_posts); + + // let end = top_posts.len() < self.cursor.limit as usize; + // let chunk_stream = top_posts + // .into_iter() + // .map(move |item| get_post_uid(self.canisters, item.0, item.1)) + // .collect::<FuturesOrdered<_>>() + // .filter_map(|res| async { res.transpose() }) + // .chunks(chunks); + + // Ok(FetchVideosRes { + // posts_stream: Box::pin(chunk_stream), + // end, + // }) + // } + + // // Empty res + // #[cfg(not(feature = "hydrate"))] + // { + // leptos::logging::log!("not hydrate"); + use crate::utils::local_feed_impl::get_next_feed; + + let user_canister_principal = self.canisters.user_canister(); + + let top_posts_fut = get_next_feed(); + + let top_posts = match top_posts_fut.await { + Ok(top_posts) => top_posts, + Err(e) => { + leptos::logging::log!("error fetching posts: {:?}", e); + return Ok(FetchVideosRes { + posts_stream: Box::pin(futures::stream::empty()), + end: true, + }) + } + }; + leptos::logging::log!("after first ret : top_posts : {:?}", top_posts); + + let end = false; + let chunk_stream = top_posts + .into_iter() + .map(move |item| get_post_uid(self.canisters, item.0, item.1)) + .collect::<FuturesOrdered<_>>() + .filter_map(|res| async { res.transpose() }) + .chunks(chunks); + + Ok(FetchVideosRes { + posts_stream: Box::pin(chunk_stream), + end, + }) + + // Ok(FetchVideosRes { + // posts_stream: Box::pin(futures::stream::empty()), + // end: true, + // }) + // } } } + diff --git a/ssr/src/state/canisters.rs b/ssr/src/state/canisters.rs index 9d7c76be..b0766073 100644 --- a/ssr/src/state/canisters.rs +++ b/ssr/src/state/canisters.rs @@ -72,9 +72,9 @@ impl Canisters<true> { .expect("Authenticated canisters must have an identity") } - pub fn user_canister(&self) -> Principal { - self.user_canister - } + // pub fn user_canister(&self) -> Principal { + // self.user_canister + // } pub async fn authenticated_user(&self) -> Result<IndividualUserTemplate<'_>, AgentError> { self.individual_user(self.user_canister).await @@ -94,6 +94,10 @@ impl Canisters<true> { } impl<const A: bool> Canisters<A> { + pub fn user_canister(&self) -> Principal { + self.user_canister + } + pub async fn post_cache(&self) -> Result<PostCache<'_>, AgentError> { let agent = self.agent.get_agent().await?; Ok(PostCache(POST_CACHE_ID, agent)) diff --git a/ssr/src/state/mod.rs b/ssr/src/state/mod.rs index 83dfd93f..c1b39880 100644 --- a/ssr/src/state/mod.rs +++ b/ssr/src/state/mod.rs @@ -8,7 +8,7 @@ pub mod local_storage; #[cfg(feature = "ssr")] pub mod server { - use crate::{auth::server_impl::store::KVStoreImpl, utils::ml_feed::MLFeedGRPCChannel}; + use crate::auth::server_impl::store::KVStoreImpl; use super::canisters::Canisters; use axum::extract::FromRef; diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index bca1a3c4..e8acaa13 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -1,105 +1,222 @@ +use crate::consts::ML_FEED_GRPC_URL; use candid::Principal; use leptos::RwSignal; -use tonic::transport::Channel; -use crate::consts::ML_FEED_GRPC_URL; +use tonic_web_wasm_client::Client; use super::types::PostId; -#[derive(Clone, Default)] -pub struct MLFeed { - pub channel: Option<Channel>, -} +use crate::utils::ml_feed::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; +use leptos::*; +pub mod ml_feed_proto { + tonic_2::include_proto!("ml_feed"); +} -pub async fn init_mlfeed_grpc_channel() -> Channel { - let ml_feed_url = ML_FEED_GRPC_URL.as_ref(); - Channel::from_static(ml_feed_url) - .connect() - .await - .expect("Couldn't connect to ML feed server") +#[derive(Clone)] +pub struct MLFeed { + pub client: MlFeedClient<Client>, } -impl MLFeed { - pub async fn get_channel(&mut self) -> &Channel { - if self.channel.is_none() { - self.channel = Some(init_mlfeed_grpc_channel().await); - } - self.channel.as_ref().unwrap() +impl Default for MLFeed { + fn default() -> Self { + let ml_feed_url = "http://localhost:50051".to_string(); + let client = Client::new(ml_feed_url); + + Self { client: MlFeedClient::new(client) } } } - -#[cfg(not(feature = "local-feed"))] +// #[cfg(not(feature = "local-feed"))] pub mod ml_feed_impl { use super::*; pub async fn get_next_feed( - ml_feed_channel: MLFeed, canister_id: &Principal, limit: u32, filter_list: Vec<PostId>, - ) -> Result<Vec<PostId>, tonic::Status> { + ) -> Result<Vec<PostId>, tonic_2::Status> { - // let mut client = MLFeedServiceClient::new(self.channel.clone()); - // let request = tonic::Request::new(MLFeedRequest { - // user_id: user_id.to_string(), - // limit: limit as u32, - // }); - // let response = client.get_recommendations(request).await?; - // Ok(response.into_inner().recommendations) + // let mut ml_feed = MLFeed::default(); - Ok(vec![]) - } -} + let mut ml_feed: MLFeed = expect_context(); -#[cfg(feature = "local-feed")] -pub mod local_feed_impl { - use super::*; + let request = FeedRequest { + canister_id: canister_id.to_string(), + filter_posts: vec![], // filter_list.iter().map(|item| PostItem {post_id: item.1 as u32, canister_id: item.0.to_string()}).collect(), + num_results: limit, + }; + + let response = ml_feed.client.get_feed(request).await.map_err(|e| { + leptos::logging::log!("error fetching posts: {:?}", e); + tonic_2::Status::new(tonic_2::Code::Internal, "error fetching posts") + })?; - pub async fn get_next_feed() -> Result<Vec<PostId>, tonic::Status> { - - let posts = vec![ - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 125), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 124), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 123), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 122), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 121), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 120), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 119), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 118), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 117), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 116), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 115), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 114), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 113), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 112), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 111), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 110), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 109), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 108), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 107), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 106), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 105), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 104), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 103), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 102), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 101), - (Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 100), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 26), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 25), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 24), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 23), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 22), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 21), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 20), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 19), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 18), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 17), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 16), - (Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), 15), - ]; - - Ok(posts) + let feed_res = response.into_inner().feed; + + Ok(feed_res.iter().map(|item| (Principal::from_text(&item.canister_id).unwrap(), item.post_id as u64)).collect()) } } +// #[cfg(feature = "local-feed")] +// pub mod local_feed_impl { +// use super::*; + +// pub async fn get_next_feed() -> Result<Vec<PostId>, tonic_2::Status> { +// let posts = vec![ +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 125, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 124, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 123, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 122, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 121, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 120, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 119, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 118, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 117, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 116, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 115, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 114, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 113, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 112, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 111, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 110, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 109, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 108, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 107, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 106, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 105, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 104, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 103, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 102, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 101, +// ), +// ( +// Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), +// 100, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 26, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 25, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 24, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 23, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 22, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 21, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 20, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 19, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 18, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 17, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 16, +// ), +// ( +// Principal::from_text("vcqbz-sqaaa-aaaag-aesbq-cai").unwrap(), +// 15, +// ), +// ]; + +// Ok(posts) +// } +// } diff --git a/ssr/src/utils/mod.rs b/ssr/src/utils/mod.rs index 05ebb55f..54e01cc8 100644 --- a/ssr/src/utils/mod.rs +++ b/ssr/src/utils/mod.rs @@ -4,7 +4,6 @@ use web_time::{Duration, SystemTime}; pub mod event_streaming; pub mod ic; pub mod icon; -pub mod ml_feed; pub mod posts; pub mod profile; pub mod report; @@ -14,6 +13,9 @@ pub mod types; pub mod user; pub mod web; +#[cfg(feature = "hydrate")] +pub mod ml_feed; + pub fn current_epoch() -> Duration { web_time::SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -36,3 +38,121 @@ impl<T> PartialEq for MockPartialEq<T> { pub mod off_chain { tonic::include_proto!("off_chain"); } + + +// TODO: to be removed +pub mod local_feed_impl { + use candid::Principal; + use super::types::PostId; + + pub async fn get_next_feed() -> Result<Vec<PostId>, String> { + let posts = vec![ + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 125, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 124, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 123, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 122, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 121, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 120, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 119, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 118, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 117, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 116, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 115, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 114, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 113, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 112, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 111, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 110, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 109, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 108, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 107, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 106, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 105, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 104, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 103, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 102, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 101, + ), + ( + Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), + 100, + ), + ]; + + Ok(posts) + } +} diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 0f4cf19d..05b2c007 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -102,12 +102,12 @@ pub async fn get_post_uid<const AUTH: bool>( { Ok(p) => p, Err(e) => { - log::warn!("failed to get post details: {}, skipping", e); + log::warn!("failed to get post details for {} {}: {}, skipping", user_canister.to_string(), post_id, e); return Ok(None); } }; - // TODO: Add this filter + // TODO: Add this filter in new method // if post_details.status == PostStatus::BannedDueToUserReporting { // return Ok(None); // } diff --git a/ssr/src/utils/types.rs b/ssr/src/utils/types.rs index d58c6676..e38f2ff9 100644 --- a/ssr/src/utils/types.rs +++ b/ssr/src/utils/types.rs @@ -1 +1,3 @@ +use candid::Principal; + pub type PostId = (Principal, u64); From 1ded44249d27dfdf3d1c2cd08b6f9796fc9c9a34 Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Tue, 6 Aug 2024 19:56:30 +0530 Subject: [PATCH 3/9] local feed testing --- ssr/Cargo.toml | 6 +- ssr/build.rs | 12 ++- ssr/contracts | 2 +- ssr/src/app.rs | 2 +- ssr/src/component/scrolling_post_view.rs | 5 +- ssr/src/consts/mod.rs | 3 +- ssr/src/page/post_view/mod.rs | 14 +-- ssr/src/page/post_view/video_iter.rs | 67 +++---------- ssr/src/page/root.rs | 30 +++++- ssr/src/utils/ml_feed/mod.rs | 94 +++++++++++------ ssr/src/utils/mod.rs | 122 +---------------------- 11 files changed, 134 insertions(+), 223 deletions(-) diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 74a0e6e3..b56ce3ce 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -86,7 +86,7 @@ gob-cloudflare = { git = "https://github.com/go-bazzinga/gob-cloudflare", rev = yral-metadata-client = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } yral-metadata-types = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } gloo-utils = { version = "0.2.0", features = ["serde"] } -tonic = { version = "0.12.1", features = [ +tonic = { version = "0.12.0", features = [ "tls", "tls-webpki-roots", ], optional = true } @@ -99,7 +99,7 @@ tonic-web-wasm-client = { version = "0.6" } [dependencies.tonic_2] package = "tonic" -version = "0.12.1" +version = "0.12.0" optional = true default-features = false features = ["prost", "codegen"] @@ -114,7 +114,7 @@ anyhow = "1.0.86" [build-dependencies.tonic_build_2] package = "tonic-build" -version = "0.12" +version = "0.12.1" default-features = false features = ["prost"] diff --git a/ssr/build.rs b/ssr/build.rs index 60d48143..4bc3f468 100644 --- a/ssr/build.rs +++ b/ssr/build.rs @@ -128,13 +128,23 @@ mod build_common { fn build_gprc_client() -> Result<()> { let ml_feed_proto = "contracts/projects/ml_feed/ml_feed.proto"; - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let mut out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + tonic_build::configure() + .build_client(true) + .build_server(false) + .out_dir(out_dir.clone()) + .compile(&[ml_feed_proto], &["proto"])?; + + out_dir = out_dir.join("grpc-web"); + fs::create_dir_all(&out_dir)?; tonic_build_2::configure() .build_client(true) .build_server(false) .out_dir(out_dir) .compile(&[ml_feed_proto], &["proto"])?; + Ok(()) } diff --git a/ssr/contracts b/ssr/contracts index 50066890..3e8f5911 160000 --- a/ssr/contracts +++ b/ssr/contracts @@ -1 +1 @@ -Subproject commit 50066890f6f8851538a2ccecd03cbe3ffd236c2c +Subproject commit 3e8f5911f9fba6746121b7847243d5d0da3748ee diff --git a/ssr/src/app.rs b/ssr/src/app.rs index c72caca6..82113bc7 100644 --- a/ssr/src/app.rs +++ b/ssr/src/app.rs @@ -69,7 +69,7 @@ pub fn App() -> impl IntoView { #[cfg(feature = "hydrate")] { - use crate::utils::ml_feed::MLFeed; + use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; provide_context(MLFeed::default()); } diff --git a/ssr/src/component/scrolling_post_view.rs b/ssr/src/component/scrolling_post_view.rs index 7d3d78a1..02a35d37 100644 --- a/ssr/src/component/scrolling_post_view.rs +++ b/ssr/src/component/scrolling_post_view.rs @@ -30,7 +30,8 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I <For each=move || video_queue().into_iter().enumerate() - key=move |(_, details)| (details.canister_id, details.post_id) + // TODO: change it back + key=move |(qidx, details)| (qidx.clone(), details.canister_id, details.post_id) children=move |(queue_idx, _details)| { let container_ref = create_node_ref::<html::Div>(); let next_videos = fetch_next_videos.clone(); @@ -74,7 +75,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I <div _ref=container_ref class="snap-always snap-end w-full h-full"> <Show when=show_video> <BgView video_queue current_idx idx=queue_idx> - <VideoView video_queue current_idx idx=queue_idx muted/> + <VideoView video_queue current_idx idx=queue_idx muted /> </BgView> </Show> </div> diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 39046978..321a2d9e 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -26,8 +26,7 @@ pub static OFF_CHAIN_AGENT_GRPC_URL: Lazy<Url> = pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); -pub static ML_FEED_GRPC_URL: Lazy<Url> = - Lazy::new(|| Url::parse("http://127.0.0.1:50051").unwrap()); +pub const ML_FEED_GRPC_URL: &str = "http://localhost:50051"; pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index 9e96df37..53f528c7 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -185,11 +185,11 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { } let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); - // let ml_feed: MLFeed = expect_context(); + // #[cfg(feature = "hydrate")] // { - // use crate::utils::ml_feed::ml_feed_impl::get_next_feed; + // use crate::utils::ml_feed::ml_feed_grpcweb::get_next_feed; // leptos::spawn_local(async move { @@ -210,14 +210,11 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let unauth_canisters = unauth_canisters(); let chunks = if let Some(canisters) = auth_canisters.as_ref() { - leptos::logging::log!("auth_canisters: yo"); let fetch_stream = VideoFetchStream::new(canisters, cursor); - fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled).await // fetch_post_uids_ml_feed_chunked + fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled, video_queue.get_untracked()).await // fetch_post_uids_ml_feed_chunked } else { - leptos::logging::log!("unauth_canisters: yo"); - // return; let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); - fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled).await // fetch_post_uids_chunked + fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled, video_queue.get_untracked()).await // fetch_post_uids_chunked }; let res = try_or_redirect!(chunks); @@ -229,14 +226,11 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { for uid in chunk { let uid = try_or_redirect!(uid); q.push(uid); - - leptos::logging::log!("queue len: {:?}, cur_idc: {:?}", q.len(), current_idx.get_untracked()); } }); } if res.end || cnt >= 8 { queue_end.try_set(res.end); - leptos::logging::log!("breaking: queue_end {:?}", queue_end.get_untracked()); break; } fetch_cursor.try_update(|c| c.advance()); diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 27a3a004..c3db5cd1 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -89,53 +89,18 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { self, chunks: usize, allow_nsfw: bool, + video_queue: Vec<PostDetails>, ) -> Result<FetchVideosRes<'a>, PostViewError> { - // #[cfg(feature = "hydrate")] - // { - // leptos::logging::log!("in hydrate"); - - // use crate::utils::ml_feed::ml_feed_impl::get_next_feed; - - // let user_canister_principal = self.canisters.user_canister(); - - // let top_posts_fut = get_next_feed(&user_canister_principal, self.cursor.limit as u32, vec![]); - - // let top_posts = match top_posts_fut.await { - // Ok(top_posts) => top_posts, - // Err(e) => { - // leptos::logging::log!("error fetching posts: {:?}", e); - // return Ok(FetchVideosRes { - // posts_stream: Box::pin(futures::stream::empty()), - // end: true, - // }) - // } - // }; - // leptos::logging::log!("in hydrate - after first ret : top_posts : {:?}", top_posts); - - // let end = top_posts.len() < self.cursor.limit as usize; - // let chunk_stream = top_posts - // .into_iter() - // .map(move |item| get_post_uid(self.canisters, item.0, item.1)) - // .collect::<FuturesOrdered<_>>() - // .filter_map(|res| async { res.transpose() }) - // .chunks(chunks); - - // Ok(FetchVideosRes { - // posts_stream: Box::pin(chunk_stream), - // end, - // }) - // } - - // // Empty res - // #[cfg(not(feature = "hydrate"))] - // { - // leptos::logging::log!("not hydrate"); - use crate::utils::local_feed_impl::get_next_feed; + #[cfg(feature = "hydrate")] + { + use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; + use leptos::expect_context; let user_canister_principal = self.canisters.user_canister(); + let mut ml_feed: MLFeed = expect_context(); - let top_posts_fut = get_next_feed(); + let top_posts_fut = ml_feed.get_next_feed(&user_canister_principal, self.cursor.limit as u32, video_queue); let top_posts = match top_posts_fut.await { Ok(top_posts) => top_posts, @@ -147,9 +112,8 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { }) } }; - leptos::logging::log!("after first ret : top_posts : {:?}", top_posts); - let end = false; + let end = top_posts.len() < self.cursor.limit as usize; let chunk_stream = top_posts .into_iter() .map(move |item| get_post_uid(self.canisters, item.0, item.1)) @@ -157,16 +121,17 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { .filter_map(|res| async { res.transpose() }) .chunks(chunks); - Ok(FetchVideosRes { + return Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, - }) + }); + } - // Ok(FetchVideosRes { - // posts_stream: Box::pin(futures::stream::empty()), - // end: true, - // }) - // } + Ok(FetchVideosRes { + posts_stream: Box::pin(futures::stream::empty()), + end: true, + }) + } } diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index ff8d437a..0a0205dd 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -35,9 +35,35 @@ async fn get_top_post_id() -> Result<Option<(Principal, u64)>, ServerFnError> { Ok(Some((top_item.publisher_canister_id, top_item.post_id))) } + +#[server] +async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnError> { + use crate::utils::ml_feed::ml_feed_grpc::get_start_feed; + + let canisters = unauth_canisters(); + let user_canister_principal = canisters.user_canister(); + let top_posts_fut = get_start_feed(&user_canister_principal, 1, vec![]); + + let top_items = match top_posts_fut.await + { + Ok(top_posts) => top_posts, + Err(e) => { + return Err(ServerFnError::ServerError( + "failed to fetch top post ml feed".to_string(), + )); + } + }; + let Some(top_item) = top_items.first() else { + return Ok(None); + }; + + Ok(Some((top_item.0, top_item.1))) +} + + #[component] pub fn RootPage() -> impl IntoView { - let target_post = create_resource(|| (), |_| get_top_post_id()); + let target_post = create_resource(|| (), |_| get_top_post_id_mlfeed()); view! { <Suspense fallback=FullScreenSpinner> @@ -52,7 +78,7 @@ pub fn RootPage() -> impl IntoView { Ok(None) => "/error?err=No Posts Found".to_string(), Err(e) => format!("/error?err={e}"), }; - view! { <Redirect path=url/> } + view! { <Redirect path=url /> } }) }} diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index e8acaa13..a49c8784 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -1,54 +1,89 @@ use crate::consts::ML_FEED_GRPC_URL; use candid::Principal; use leptos::RwSignal; -use tonic_web_wasm_client::Client; use super::types::PostId; -use crate::utils::ml_feed::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; use leptos::*; -pub mod ml_feed_proto { - tonic_2::include_proto!("ml_feed"); -} -#[derive(Clone)] -pub struct MLFeed { - pub client: MlFeedClient<Client>, -} +#[cfg(feature = "hydrate")] +pub mod ml_feed_grpcweb { + use crate::utils::posts::PostDetails; + use crate::utils::ml_feed::ml_feed_grpcweb::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; + use tonic_web_wasm_client::Client; + use super::*; -impl Default for MLFeed { - fn default() -> Self { - let ml_feed_url = "http://localhost:50051".to_string(); - let client = Client::new(ml_feed_url); + pub mod ml_feed_proto { + tonic_2::include_proto!("ml_feed"); + } + + #[derive(Clone)] + pub struct MLFeed { + pub client: MlFeedClient<Client>, + } + + impl Default for MLFeed { + fn default() -> Self { + let client = Client::new(ML_FEED_GRPC_URL.to_string()); + + Self { client: MlFeedClient::new(client) } + } + } - Self { client: MlFeedClient::new(client) } + impl MLFeed { + pub async fn get_next_feed( + mut self, + canister_id: &Principal, + limit: u32, + filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic_2::Status> { + + let request = FeedRequest { + canister_id: canister_id.to_string(), + filter_posts: filter_list.iter().map(|item| PostItem {post_id: item.post_id as u32, canister_id: item.canister_id.to_string(), video_id: item.uid.clone()}).collect(), + num_results: limit, + }; + + let response = self.client.get_feed(request).await.map_err(|e| { + tonic_2::Status::new(tonic_2::Code::Internal, "error fetching posts") + })?; + + let feed_res = response.into_inner().feed; + + Ok(feed_res.iter().map(|item| (Principal::from_text(&item.canister_id).unwrap(), item.post_id as u64)).collect()) + } } } -// #[cfg(not(feature = "local-feed"))] -pub mod ml_feed_impl { + + +#[cfg(feature = "ssr")] +pub mod ml_feed_grpc { + use crate::utils::posts::PostDetails; + use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; use super::*; - pub async fn get_next_feed( + pub mod ml_feed_proto { + include!(concat!(env!("OUT_DIR"), "/grpc-ssr/ml_feed.rs")); + } + + pub async fn get_start_feed( canister_id: &Principal, limit: u32, - filter_list: Vec<PostId>, - ) -> Result<Vec<PostId>, tonic_2::Status> { - - // let mut ml_feed = MLFeed::default(); - - let mut ml_feed: MLFeed = expect_context(); + filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic::Status> { - let request = FeedRequest { + let mut client = MlFeedClient::connect(ML_FEED_GRPC_URL).await.unwrap(); + + let request = tonic::Request::new(FeedRequest { canister_id: canister_id.to_string(), - filter_posts: vec![], // filter_list.iter().map(|item| PostItem {post_id: item.1 as u32, canister_id: item.0.to_string()}).collect(), + filter_posts: filter_list.iter().map(|item| PostItem {post_id: item.post_id as u32, canister_id: item.canister_id.to_string(), video_id: item.uid.clone()}).collect(), num_results: limit, - }; + }); - let response = ml_feed.client.get_feed(request).await.map_err(|e| { - leptos::logging::log!("error fetching posts: {:?}", e); - tonic_2::Status::new(tonic_2::Code::Internal, "error fetching posts") + let response = client.get_feed(request).await.map_err(|e| { + tonic::Status::new(tonic::Code::Internal, "error fetching posts") })?; let feed_res = response.into_inner().feed; @@ -57,6 +92,7 @@ pub mod ml_feed_impl { } } +// TODO: remove // #[cfg(feature = "local-feed")] // pub mod local_feed_impl { // use super::*; diff --git a/ssr/src/utils/mod.rs b/ssr/src/utils/mod.rs index 54e01cc8..05ebb55f 100644 --- a/ssr/src/utils/mod.rs +++ b/ssr/src/utils/mod.rs @@ -4,6 +4,7 @@ use web_time::{Duration, SystemTime}; pub mod event_streaming; pub mod ic; pub mod icon; +pub mod ml_feed; pub mod posts; pub mod profile; pub mod report; @@ -13,9 +14,6 @@ pub mod types; pub mod user; pub mod web; -#[cfg(feature = "hydrate")] -pub mod ml_feed; - pub fn current_epoch() -> Duration { web_time::SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -38,121 +36,3 @@ impl<T> PartialEq for MockPartialEq<T> { pub mod off_chain { tonic::include_proto!("off_chain"); } - - -// TODO: to be removed -pub mod local_feed_impl { - use candid::Principal; - use super::types::PostId; - - pub async fn get_next_feed() -> Result<Vec<PostId>, String> { - let posts = vec![ - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 125, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 124, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 123, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 122, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 121, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 120, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 119, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 118, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 117, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 116, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 115, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 114, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 113, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 112, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 111, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 110, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 109, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 108, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 107, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 106, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 105, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 104, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 103, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 102, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 101, - ), - ( - Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), - 100, - ), - ]; - - Ok(posts) - } -} From cf83abc0e84b7f15208a7c55aef4e6e5fb8e75ff Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Wed, 7 Aug 2024 19:34:29 +0530 Subject: [PATCH 4/9] ml feed --- Cargo.lock | 132 ++++++++++++++++++++++++++-------- ssr/Cargo.toml | 14 ++-- ssr/src/consts/mod.rs | 2 +- ssr/src/lib.rs | 2 + ssr/src/page/post_view/mod.rs | 18 ----- ssr/src/page/root.rs | 6 +- ssr/src/utils/ml_feed/mod.rs | 101 ++++++++++++++++++-------- ssr/src/utils/posts.rs | 2 +- 8 files changed, 192 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d9f2c40..d444ee88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,34 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.30", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.5" @@ -232,7 +260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.3", "axum-macros", "bytes", "futures-util", @@ -261,6 +289,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.3" @@ -288,8 +333,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum", - "axum-core", + "axum 0.7.5", + "axum-core 0.4.3", "bytes", "cookie", "futures-util", @@ -2533,7 +2578,7 @@ name = "hot-or-not-web-leptos-ssr" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.7.5", "axum-extra", "bb8", "bb8-redis", @@ -2566,6 +2611,7 @@ dependencies = [ "log", "once_cell", "openidconnect", + "prost 0.12.6", "prost 0.13.1", "rand_chacha", "redb", @@ -2579,9 +2625,10 @@ dependencies = [ "testcontainers", "thiserror", "tokio", - "tonic", + "tonic 0.11.0", + "tonic 0.12.1", "tonic-build 0.11.0", - "tonic-build 0.12.1", + "tonic-build 0.12.0", "tonic-web-wasm-client", "tower", "tower-http", @@ -2793,15 +2840,14 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 1.4.1", - "hyper-util", + "hyper 0.14.30", "pin-project-lite", "tokio", - "tower-service", + "tokio-io-timeout", ] [[package]] @@ -3428,7 +3474,7 @@ version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3923af454949eb7a5ea9a89d5fdc4d21e4850eb8d47d942d35355e5079444867" dependencies = [ - "axum", + "axum 0.7.5", "cfg-if", "futures", "http-body-util", @@ -5046,7 +5092,6 @@ version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "log", "once_cell", "ring", "rustls-pki-types", @@ -5418,7 +5463,7 @@ version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9aaae169927ef701a4734d680adcb08f13269c9f0af822bf175d681fe56d65c" dependencies = [ - "axum", + "axum 0.7.5", "bytes", "ciborium", "const_format", @@ -5898,6 +5943,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.4.0" @@ -6033,37 +6088,56 @@ dependencies = [ "winnow 0.6.15", ] +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.30", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.12.6", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots 0.26.3", +] + [[package]] name = "tonic" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" dependencies = [ - "async-stream", "async-trait", - "axum", "base64 0.22.1", "bytes", - "h2 0.4.5", "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", - "hyper-timeout", - "hyper-util", "percent-encoding", "pin-project", "prost 0.13.1", - "rustls-pemfile 2.1.2", - "socket2", - "tokio", - "tokio-rustls 0.26.0", "tokio-stream", - "tower", "tower-layer", "tower-service", "tracing", - "webpki-roots 0.26.3", ] [[package]] @@ -6081,9 +6155,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.12.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568392c5a2bd0020723e3f387891176aabafe36fd9fcd074ad309dfa0c8eb964" +checksum = "690943cc223adcdd67bb597a2e573ead1b88e999ba37528fe8e6356bf44b29b6" dependencies = [ "prettyplease", "proc-macro2", @@ -6109,7 +6183,7 @@ dependencies = [ "js-sys", "pin-project", "thiserror", - "tonic", + "tonic 0.12.1", "tower-service", "wasm-bindgen", "wasm-bindgen-futures", diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index b56ce3ce..9e45fbec 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -86,11 +86,11 @@ gob-cloudflare = { git = "https://github.com/go-bazzinga/gob-cloudflare", rev = yral-metadata-client = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } yral-metadata-types = { git = "https://github.com/go-bazzinga/yral-metadata", rev = "c394bf9af3f32d81c1ac50b966c25dafafa2545b", default-features = false } gloo-utils = { version = "0.2.0", features = ["serde"] } -tonic = { version = "0.12.0", features = [ +tonic = { version = "0.11.0", features = [ "tls", "tls-webpki-roots", ], optional = true } -prost = { version = "0.13.1", optional = true } +prost = { version = "0.12.4", optional = true } hmac = { version = "0.12.1", optional = true } wasm-bindgen-futures = { version = "0.4.42", optional = true } testcontainers = { version = "0.20.0", optional = true } @@ -104,6 +104,12 @@ optional = true default-features = false features = ["prost", "codegen"] + +[dependencies.prost_2] +package = "prost" +version = "0.13.0" +optional = true + [build-dependencies] serde = { version = "1.0", features = ["derive"] } candid_parser = "0.1.1" @@ -114,7 +120,7 @@ anyhow = "1.0.86" [build-dependencies.tonic_build_2] package = "tonic-build" -version = "0.12.1" +version = "=0.12.0" default-features = false features = ["prost"] @@ -129,7 +135,7 @@ hydrate = [ "dep:rand_chacha", "dep:wasm-bindgen-futures", "tonic_2", - "prost", + "prost_2", ] ssr = [ "dep:axum", diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 321a2d9e..42183f75 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -26,7 +26,7 @@ pub static OFF_CHAIN_AGENT_GRPC_URL: Lazy<Url> = pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); -pub const ML_FEED_GRPC_URL: &str = "http://localhost:50051"; +pub const ML_FEED_GRPC_URL: &str = "https://yral-ml-feed-server-staging.fly.dev:443"; // "http://localhost:50051"; pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/lib.rs b/ssr/src/lib.rs index 0ca490a6..3ea2a6de 100644 --- a/ssr/src/lib.rs +++ b/ssr/src/lib.rs @@ -14,6 +14,8 @@ pub mod page; pub mod state; pub mod utils; +#[cfg(feature = "hydrate")] +extern crate prost_2 as prost; #[cfg(feature = "hydrate")] extern crate tonic_2 as tonic; diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index 53f528c7..3ca923e1 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -170,7 +170,6 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { return; } f.start = 1; - f.limit = 1; }); video_queue.update_untracked(|v| { if v.len() > 1 { @@ -186,20 +185,6 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); - - // #[cfg(feature = "hydrate")] - // { - // use crate::utils::ml_feed::ml_feed_grpcweb::get_next_feed; - - // leptos::spawn_local(async move { - - // let temp_res = get_next_feed(&Principal::from_text("76qol-iiaaa-aaaak-qelkq-cai").unwrap(), 10, vec![]).await; - - // logging::log!("temp_res: {:?}", temp_res); - // }); - - // } - let fetch_video_action = create_action(move |_|{ async move { loop { let Some(cursor) = fetch_cursor.try_get_untracked() else { @@ -233,10 +218,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { queue_end.try_set(res.end); break; } - fetch_cursor.try_update(|c| c.advance()); } - - fetch_cursor.try_update(|c| c.advance()); }}); create_effect(move |_| { if !recovering_state.get_untracked() { diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index 0a0205dd..0398357f 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -35,7 +35,6 @@ async fn get_top_post_id() -> Result<Option<(Principal, u64)>, ServerFnError> { Ok(Some((top_item.publisher_canister_id, top_item.post_id))) } - #[server] async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnError> { use crate::utils::ml_feed::ml_feed_grpc::get_start_feed; @@ -44,10 +43,10 @@ async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnEr let user_canister_principal = canisters.user_canister(); let top_posts_fut = get_start_feed(&user_canister_principal, 1, vec![]); - let top_items = match top_posts_fut.await - { + let top_items = match top_posts_fut.await { Ok(top_posts) => top_posts, Err(e) => { + log::error!("failed to fetch top post ml feed: {:?}", e); return Err(ServerFnError::ServerError( "failed to fetch top post ml feed".to_string(), )); @@ -60,7 +59,6 @@ async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnEr Ok(Some((top_item.0, top_item.1))) } - #[component] pub fn RootPage() -> impl IntoView { let target_post = create_resource(|| (), |_| get_top_post_id_mlfeed()); diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index a49c8784..8affaea6 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -6,28 +6,31 @@ use super::types::PostId; use leptos::*; - #[cfg(feature = "hydrate")] pub mod ml_feed_grpcweb { + use super::*; + use crate::utils::ml_feed::ml_feed_grpcweb::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, PostItem, + }; use crate::utils::posts::PostDetails; - use crate::utils::ml_feed::ml_feed_grpcweb::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; use tonic_web_wasm_client::Client; - use super::*; pub mod ml_feed_proto { - tonic_2::include_proto!("ml_feed"); + include!(concat!(env!("OUT_DIR"), "/grpc-web/ml_feed.rs")); } - + #[derive(Clone)] pub struct MLFeed { pub client: MlFeedClient<Client>, } - + impl Default for MLFeed { fn default() -> Self { let client = Client::new(ML_FEED_GRPC_URL.to_string()); - - Self { client: MlFeedClient::new(client) } + + Self { + client: MlFeedClient::new(client), + } } } @@ -38,57 +41,99 @@ pub mod ml_feed_grpcweb { limit: u32, filter_list: Vec<PostDetails>, ) -> Result<Vec<PostId>, tonic_2::Status> { - let request = FeedRequest { canister_id: canister_id.to_string(), - filter_posts: filter_list.iter().map(|item| PostItem {post_id: item.post_id as u32, canister_id: item.canister_id.to_string(), video_id: item.uid.clone()}).collect(), + filter_posts: filter_list + .iter() + .map(|item| PostItem { + post_id: item.post_id as u32, + canister_id: item.canister_id.to_string(), + video_id: item.uid.clone(), + }) + .collect(), num_results: limit, }; - - let response = self.client.get_feed(request).await.map_err(|e| { - tonic_2::Status::new(tonic_2::Code::Internal, "error fetching posts") + + let response = self.client.get_feed(request).await.map_err(|e| { + tonic_2::Status::new( + tonic_2::Code::Internal, + format!("Error fetching posts: {:?}", e), + ) })?; - + let feed_res = response.into_inner().feed; - - Ok(feed_res.iter().map(|item| (Principal::from_text(&item.canister_id).unwrap(), item.post_id as u64)).collect()) + + Ok(feed_res + .iter() + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect()) } } } - - #[cfg(feature = "ssr")] pub mod ml_feed_grpc { - use crate::utils::posts::PostDetails; - use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ml_feed_client::MlFeedClient, FeedRequest, PostItem}; use super::*; + use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, PostItem, + }; + use crate::utils::posts::PostDetails; + use tonic::transport::Channel; pub mod ml_feed_proto { - include!(concat!(env!("OUT_DIR"), "/grpc-ssr/ml_feed.rs")); + tonic::include_proto!("ml_feed"); } - + pub async fn get_start_feed( canister_id: &Principal, limit: u32, filter_list: Vec<PostDetails>, ) -> Result<Vec<PostId>, tonic::Status> { + let channel = Channel::from_static("https://yral-ml-feed-server-staging.fly.dev:443") + .connect() + .await + .expect("Couldn't connect to ML feed server"); + + // let mut client = MlFeedClient::connect(ML_FEED_GRPC_URL).await.unwrap(); + + let mut client = MlFeedClient::new(channel); - let mut client = MlFeedClient::connect(ML_FEED_GRPC_URL).await.unwrap(); - let request = tonic::Request::new(FeedRequest { canister_id: canister_id.to_string(), - filter_posts: filter_list.iter().map(|item| PostItem {post_id: item.post_id as u32, canister_id: item.canister_id.to_string(), video_id: item.uid.clone()}).collect(), + filter_posts: filter_list + .iter() + .map(|item| PostItem { + post_id: item.post_id as u32, + canister_id: item.canister_id.to_string(), + video_id: item.uid.clone(), + }) + .collect(), num_results: limit, }); - let response = client.get_feed(request).await.map_err(|e| { - tonic::Status::new(tonic::Code::Internal, "error fetching posts") + let response = client.get_feed(request).await.map_err(|e| { + tonic::Status::new( + tonic::Code::Internal, + format!("error fetching posts: {:?}", e), + ) })?; let feed_res = response.into_inner().feed; - Ok(feed_res.iter().map(|item| (Principal::from_text(&item.canister_id).unwrap(), item.post_id as u64)).collect()) + Ok(feed_res + .iter() + .map(|item| { + ( + Principal::from_text(&item.canister_id).unwrap(), + item.post_id as u64, + ) + }) + .collect()) } } diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 05b2c007..36892821 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -30,7 +30,7 @@ impl Default for FetchCursor { fn default() -> Self { Self { start: 0, - limit: 10, + limit: 25, } } } From 2a138cc8d5b6a0e1ad30fe13e7bae145f14c344c Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Wed, 7 Aug 2024 19:46:10 +0530 Subject: [PATCH 5/9] fmt --- .github/workflows/build-check.yml | 2 +- ssr/Cargo.toml | 1 + ssr/src/init/mod.rs | 2 - ssr/src/page/post_view/mod.rs | 79 +++++++++++++++++----------- ssr/src/page/post_view/video_iter.rs | 34 ++++++------ ssr/src/utils/ml_feed/mod.rs | 3 -- ssr/src/utils/posts.rs | 9 +++- 7 files changed, 75 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-check.yml b/.github/workflows/build-check.yml index 34ea2c98..e09be7c1 100644 --- a/.github/workflows/build-check.yml +++ b/.github/workflows/build-check.yml @@ -90,4 +90,4 @@ jobs: target/x86_64-unknown-linux-musl/release/hot-or-not-web-leptos-ssr target/release/hash.txt target/site - .empty \ No newline at end of file + .empty diff --git a/ssr/Cargo.toml b/ssr/Cargo.toml index 9e45fbec..765d1c63 100644 --- a/ssr/Cargo.toml +++ b/ssr/Cargo.toml @@ -89,6 +89,7 @@ gloo-utils = { version = "0.2.0", features = ["serde"] } tonic = { version = "0.11.0", features = [ "tls", "tls-webpki-roots", + "transport", # for clippy ], optional = true } prost = { version = "0.12.4", optional = true } hmac = { version = "0.12.1", optional = true } diff --git a/ssr/src/init/mod.rs b/ssr/src/init/mod.rs index c01c6a26..b542db46 100644 --- a/ssr/src/init/mod.rs +++ b/ssr/src/init/mod.rs @@ -66,8 +66,6 @@ async fn init_grpc_offchain_channel() -> tonic::transport::Channel { .expect("Couldn't connect to off-chain agent") } - - #[cfg(feature = "backend-admin")] fn init_admin_canisters() -> crate::state::admin_canisters::AdminCanisters { use crate::state::admin_canisters::AdminCanisters; diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index 3ca923e1..baa4e123 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -19,7 +19,8 @@ use crate::{ state::canisters::{unauth_canisters, Canisters}, try_or_redirect, utils::{ - posts::{get_post_uid, FetchCursor, PostDetails}, route::failure_redirect + posts::{get_post_uid, FetchCursor, PostDetails}, + route::failure_redirect, }, }; use video_iter::VideoFetchStream; @@ -185,41 +186,55 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); - let fetch_video_action = create_action(move |_|{ async move { - loop { - let Some(cursor) = fetch_cursor.try_get_untracked() else { - return; - }; - let auth_canisters = auth_canisters.get_untracked(); - let nsfw_enabled = nsfw_enabled.get_untracked(); - let unauth_canisters = unauth_canisters(); + let fetch_video_action = create_action(move |_| { + async move { + loop { + let Some(cursor) = fetch_cursor.try_get_untracked() else { + return; + }; + let auth_canisters = auth_canisters.get_untracked(); + let nsfw_enabled = nsfw_enabled.get_untracked(); + let unauth_canisters = unauth_canisters(); - let chunks = if let Some(canisters) = auth_canisters.as_ref() { - let fetch_stream = VideoFetchStream::new(canisters, cursor); - fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled, video_queue.get_untracked()).await // fetch_post_uids_ml_feed_chunked - } else { - let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); - fetch_stream.fetch_post_uids_ml_feed_chunked(3, nsfw_enabled, video_queue.get_untracked()).await // fetch_post_uids_chunked - }; + let chunks = if let Some(canisters) = auth_canisters.as_ref() { + let fetch_stream = VideoFetchStream::new(canisters, cursor); + fetch_stream + .fetch_post_uids_ml_feed_chunked( + 3, + nsfw_enabled, + video_queue.get_untracked(), + ) + .await // fetch_post_uids_ml_feed_chunked + } else { + let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); + fetch_stream + .fetch_post_uids_ml_feed_chunked( + 3, + nsfw_enabled, + video_queue.get_untracked(), + ) + .await // fetch_post_uids_chunked + }; - let res = try_or_redirect!(chunks); - let mut chunks = res.posts_stream; - let mut cnt = 0; - while let Some(chunk) = chunks.next().await { - cnt += chunk.len(); - video_queue.try_update(|q| { - for uid in chunk { - let uid = try_or_redirect!(uid); - q.push(uid); - } - }); - } - if res.end || cnt >= 8 { - queue_end.try_set(res.end); - break; + let res = try_or_redirect!(chunks); + let mut chunks = res.posts_stream; + let mut cnt = 0; + while let Some(chunk) = chunks.next().await { + cnt += chunk.len(); + video_queue.try_update(|q| { + for uid in chunk { + let uid = try_or_redirect!(uid); + q.push(uid); + } + }); + } + if res.end || cnt >= 8 { + queue_end.try_set(res.end); + break; + } } } - }}); + }); create_effect(move |_| { if !recovering_state.get_untracked() { fetch_video_action.dispatch(()); diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index c3db5cd1..4af579c3 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -6,7 +6,7 @@ use futures::{stream::FuturesOrdered, Stream, StreamExt}; use crate::{ canister::post_cache::{self, NsfwFilter}, state::canisters::Canisters, - utils::{posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}}, + utils::posts::{get_post_uid, FetchCursor, PostDetails, PostViewError}, }; pub async fn post_liked_by_me( @@ -88,20 +88,23 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { pub async fn fetch_post_uids_ml_feed_chunked( self, chunks: usize, - allow_nsfw: bool, + _allow_nsfw: bool, video_queue: Vec<PostDetails>, ) -> Result<FetchVideosRes<'a>, PostViewError> { - #[cfg(feature = "hydrate")] { use crate::utils::ml_feed::ml_feed_grpcweb::MLFeed; use leptos::expect_context; let user_canister_principal = self.canisters.user_canister(); - let mut ml_feed: MLFeed = expect_context(); + let ml_feed: MLFeed = expect_context(); + + let top_posts_fut = ml_feed.get_next_feed( + &user_canister_principal, + self.cursor.limit as u32, + video_queue, + ); - let top_posts_fut = ml_feed.get_next_feed(&user_canister_principal, self.cursor.limit as u32, video_queue); - let top_posts = match top_posts_fut.await { Ok(top_posts) => top_posts, Err(e) => { @@ -109,10 +112,10 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { return Ok(FetchVideosRes { posts_stream: Box::pin(futures::stream::empty()), end: true, - }) + }); } }; - + let end = top_posts.len() < self.cursor.limit as usize; let chunk_stream = top_posts .into_iter() @@ -120,18 +123,19 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { .collect::<FuturesOrdered<_>>() .filter_map(|res| async { res.transpose() }) .chunks(chunks); - + return Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, }); } - Ok(FetchVideosRes { - posts_stream: Box::pin(futures::stream::empty()), - end: true, - }) - + #[cfg(not(feature = "hydrate"))] + { + return Ok(FetchVideosRes { + posts_stream: Box::pin(futures::stream::empty()), + end: true, + }); + } } } - diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index 8affaea6..e837dab2 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -1,11 +1,8 @@ use crate::consts::ML_FEED_GRPC_URL; use candid::Principal; -use leptos::RwSignal; use super::types::PostId; -use leptos::*; - #[cfg(feature = "hydrate")] pub mod ml_feed_grpcweb { use super::*; diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 36892821..81bc59d0 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -2,7 +2,7 @@ use candid::Principal; use serde::{Deserialize, Serialize}; use crate::{ - canister::individual_user_template::{PostDetailsForFrontend, PostStatus}, state::canisters::Canisters, + canister::individual_user_template::PostDetailsForFrontend, state::canisters::Canisters, }; use super::profile::propic_from_principal; @@ -102,7 +102,12 @@ pub async fn get_post_uid<const AUTH: bool>( { Ok(p) => p, Err(e) => { - log::warn!("failed to get post details for {} {}: {}, skipping", user_canister.to_string(), post_id, e); + log::warn!( + "failed to get post details for {} {}: {}, skipping", + user_canister.to_string(), + post_id, + e + ); return Ok(None); } }; From 40c2bde0c34c69cbbe4db05852a3d8caf9cb26ae Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Wed, 7 Aug 2024 21:23:07 +0530 Subject: [PATCH 6/9] dedup --- ssr/src/component/scrolling_post_view.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssr/src/component/scrolling_post_view.rs b/ssr/src/component/scrolling_post_view.rs index 02a35d37..790a70ac 100644 --- a/ssr/src/component/scrolling_post_view.rs +++ b/ssr/src/component/scrolling_post_view.rs @@ -31,7 +31,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I <For each=move || video_queue().into_iter().enumerate() // TODO: change it back - key=move |(qidx, details)| (qidx.clone(), details.canister_id, details.post_id) + key=move |(_, details)| (details.canister_id, details.post_id) children=move |(queue_idx, _details)| { let container_ref = create_node_ref::<html::Div>(); let next_videos = fetch_next_videos.clone(); From 26de6a74f7c4cad1b214a4858bd17a1fdf1d7025 Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Wed, 7 Aug 2024 22:25:08 +0530 Subject: [PATCH 7/9] clippy fixes --- ssr/src/init/mod.rs | 2 ++ ssr/src/main.rs | 4 ++++ ssr/src/page/post_view/video_iter.rs | 4 ++-- ssr/src/state/mod.rs | 5 +++-- ssr/src/utils/event_streaming/mod.rs | 12 ++++++++++++ ssr/src/utils/ml_feed/mod.rs | 21 +++++++++++++++++---- ssr/src/utils/mod.rs | 1 + ssr/src/utils/report.rs | 19 ++++++++++++++++++- 8 files changed, 59 insertions(+), 9 deletions(-) diff --git a/ssr/src/init/mod.rs b/ssr/src/init/mod.rs index b542db46..caad43f4 100644 --- a/ssr/src/init/mod.rs +++ b/ssr/src/init/mod.rs @@ -54,6 +54,7 @@ fn init_google_oauth() -> openidconnect::core::CoreClient { .set_redirect_uri(RedirectUrl::new(redirect_uri).unwrap()) } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] async fn init_grpc_offchain_channel() -> tonic::transport::Channel { use crate::consts::OFF_CHAIN_AGENT_GRPC_URL; @@ -161,6 +162,7 @@ impl AppStateBuilder { cookie_key: init_cookie_key(), #[cfg(feature = "oauth-ssr")] google_oauth: init_google_oauth(), + #[cfg(not(clippy))] #[cfg(feature = "ga4")] grpc_offchain_channel: init_grpc_offchain_channel().await, }; diff --git a/ssr/src/main.rs b/ssr/src/main.rs index abb535a6..124591e6 100644 --- a/ssr/src/main.rs +++ b/ssr/src/main.rs @@ -29,6 +29,8 @@ pub async fn server_fn_handler( provide_context(app_state.cookie_key.clone()); #[cfg(feature = "oauth-ssr")] provide_context(app_state.google_oauth.clone()); + + #[cfg(not(clippy))] #[cfg(feature = "ga4")] provide_context(app_state.grpc_offchain_channel.clone()); }, @@ -54,6 +56,8 @@ pub async fn leptos_routes_handler( provide_context(app_state.cookie_key.clone()); #[cfg(feature = "oauth-ssr")] provide_context(app_state.google_oauth.clone()); + + #[cfg(not(clippy))] #[cfg(feature = "ga4")] provide_context(app_state.grpc_offchain_channel.clone()); }, diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 4af579c3..a8420ab1 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -124,10 +124,10 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { .filter_map(|res| async { res.transpose() }) .chunks(chunks); - return Ok(FetchVideosRes { + Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, - }); + }) } #[cfg(not(feature = "hydrate"))] diff --git a/ssr/src/state/mod.rs b/ssr/src/state/mod.rs index c1b39880..eb79b206 100644 --- a/ssr/src/state/mod.rs +++ b/ssr/src/state/mod.rs @@ -8,6 +8,7 @@ pub mod local_storage; #[cfg(feature = "ssr")] pub mod server { + use crate::auth::server_impl::store::KVStoreImpl; use super::canisters::Canisters; @@ -15,7 +16,6 @@ pub mod server { use axum_extra::extract::cookie::Key; use leptos::LeptosOptions; use leptos_router::RouteListing; - use tonic::transport::Channel; #[derive(FromRef, Clone)] pub struct AppState { @@ -30,7 +30,8 @@ pub mod server { pub cookie_key: Key, #[cfg(feature = "oauth-ssr")] pub google_oauth: openidconnect::core::CoreClient, + #[cfg(not(clippy))] #[cfg(feature = "ga4")] - pub grpc_offchain_channel: Channel, + pub grpc_offchain_channel: tonic::transport::Channel, } } diff --git a/ssr/src/utils/event_streaming/mod.rs b/ssr/src/utils/event_streaming/mod.rs index c627c241..75058b4e 100644 --- a/ssr/src/utils/event_streaming/mod.rs +++ b/ssr/src/utils/event_streaming/mod.rs @@ -9,6 +9,7 @@ use crate::consts::GTAG_MEASUREMENT_ID; pub mod events; +#[cfg(not(clippy))] #[cfg(feature = "ssr")] pub mod warehouse_events { tonic::include_proto!("warehouse_events"); @@ -64,6 +65,7 @@ pub fn send_event_warehouse(event_name: &str, params: &serde_json::Value) { }); } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] #[server] pub async fn stream_to_offchain_agent(event: String, params: String) -> Result<(), ServerFnError> { @@ -94,3 +96,13 @@ pub async fn stream_to_offchain_agent(event: String, params: String) -> Result<( Ok(()) } + +#[cfg(clippy)] +#[cfg(feature = "ga4")] +#[server] +pub async fn stream_to_offchain_agent( + _event: String, + _params: String, +) -> Result<(), ServerFnError> { + Ok(()) +} diff --git a/ssr/src/utils/ml_feed/mod.rs b/ssr/src/utils/ml_feed/mod.rs index e837dab2..9affc89a 100644 --- a/ssr/src/utils/ml_feed/mod.rs +++ b/ssr/src/utils/ml_feed/mod.rs @@ -76,21 +76,24 @@ pub mod ml_feed_grpcweb { #[cfg(feature = "ssr")] pub mod ml_feed_grpc { use super::*; - use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ - ml_feed_client::MlFeedClient, FeedRequest, PostItem, - }; use crate::utils::posts::PostDetails; - use tonic::transport::Channel; + #[cfg(not(clippy))] pub mod ml_feed_proto { tonic::include_proto!("ml_feed"); } + #[cfg(not(clippy))] pub async fn get_start_feed( canister_id: &Principal, limit: u32, filter_list: Vec<PostDetails>, ) -> Result<Vec<PostId>, tonic::Status> { + use crate::utils::ml_feed::ml_feed_grpc::ml_feed_proto::{ + ml_feed_client::MlFeedClient, FeedRequest, PostItem, + }; + use tonic::transport::Channel; + let channel = Channel::from_static("https://yral-ml-feed-server-staging.fly.dev:443") .connect() .await @@ -132,6 +135,16 @@ pub mod ml_feed_grpc { }) .collect()) } + + // empty function + #[cfg(clippy)] + pub async fn get_start_feed( + _canister_id: &Principal, + _limit: u32, + _filter_list: Vec<PostDetails>, + ) -> Result<Vec<PostId>, tonic::Status> { + Ok(vec![]) + } } // TODO: remove diff --git a/ssr/src/utils/mod.rs b/ssr/src/utils/mod.rs index 05ebb55f..a74aac42 100644 --- a/ssr/src/utils/mod.rs +++ b/ssr/src/utils/mod.rs @@ -32,6 +32,7 @@ impl<T> PartialEq for MockPartialEq<T> { } } +#[cfg(not(clippy))] #[cfg(all(feature = "ga4", feature = "ssr"))] pub mod off_chain { tonic::include_proto!("off_chain"); diff --git a/ssr/src/utils/report.rs b/ssr/src/utils/report.rs index 79d35296..36f55ac5 100644 --- a/ssr/src/utils/report.rs +++ b/ssr/src/utils/report.rs @@ -1,6 +1,6 @@ use std::{env, fmt::Display}; -use leptos::{expect_context, server, ServerFnError}; +use leptos::{server, ServerFnError}; pub enum ReportOption { Nudity, @@ -22,6 +22,7 @@ impl ReportOption { } } +#[cfg(not(clippy))] #[cfg(feature = "ga4")] #[server] pub async fn send_report_offchain( @@ -34,6 +35,7 @@ pub async fn send_report_offchain( video_url: String, ) -> Result<(), ServerFnError> { use crate::utils::off_chain; + use leptos::expect_context; use tonic::metadata::MetadataValue; use tonic::transport::Channel; use tonic::Request; @@ -68,3 +70,18 @@ pub async fn send_report_offchain( Ok(()) } + +#[cfg(clippy)] +#[cfg(feature = "ga4")] +#[server] +pub async fn send_report_offchain( + _reporter_id: String, + _publisher_id: String, + _publisher_canister_id: String, + _post_id: String, + _video_id: String, + _reason: String, + _video_url: String, +) -> Result<(), ServerFnError> { + Ok(()) +} From 38c0ffa5dc046f10257d17c8f4d5901b31d2f38e Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Thu, 8 Aug 2024 22:01:47 +0530 Subject: [PATCH 8/9] new logic changes --- ssr/src/component/scrolling_post_view.rs | 2 +- ssr/src/consts/mod.rs | 2 +- ssr/src/page/post_view/mod.rs | 86 +++++++++++------------- ssr/src/page/post_view/video_iter.rs | 48 +++++++++++-- ssr/src/page/root.rs | 43 ++++++------ ssr/src/utils/posts.rs | 19 +++--- ssr/src/utils/types.rs | 26 +++++++ 7 files changed, 143 insertions(+), 83 deletions(-) diff --git a/ssr/src/component/scrolling_post_view.rs b/ssr/src/component/scrolling_post_view.rs index 790a70ac..e3b6895e 100644 --- a/ssr/src/component/scrolling_post_view.rs +++ b/ssr/src/component/scrolling_post_view.rs @@ -49,7 +49,7 @@ pub fn ScrollingPostView<F: Fn() -> V + Clone + 'static, V, O: Fn() -> IV, IV: I return; } if video_queue.with_untracked(|q| q.len()).saturating_sub(queue_idx) - <= 10 + <= 15 { next_videos.as_ref().map(|nv| { nv() }); } diff --git a/ssr/src/consts/mod.rs b/ssr/src/consts/mod.rs index 42183f75..8867fc82 100644 --- a/ssr/src/consts/mod.rs +++ b/ssr/src/consts/mod.rs @@ -26,7 +26,7 @@ pub static OFF_CHAIN_AGENT_GRPC_URL: Lazy<Url> = pub static GTAG_MEASUREMENT_ID: Lazy<&str> = Lazy::new(|| "G-PLNNETMSLM"); pub static DOWNLOAD_UPLOAD_SERVICE: Lazy<Url> = Lazy::new(|| Url::parse("https://download-upload-service.fly.dev").unwrap()); -pub const ML_FEED_GRPC_URL: &str = "https://yral-ml-feed-server-staging.fly.dev:443"; // "http://localhost:50051"; +pub const ML_FEED_GRPC_URL: &str = "https://yral-ml-feed-server-staging-v2.fly.dev:443"; pub mod social { pub const TELEGRAM: &str = "https://t.me/+c-LTX0Cp-ENmMzI1"; diff --git a/ssr/src/page/post_view/mod.rs b/ssr/src/page/post_view/mod.rs index baa4e123..c6da2b0d 100644 --- a/ssr/src/page/post_view/mod.rs +++ b/ssr/src/page/post_view/mod.rs @@ -23,7 +23,7 @@ use crate::{ route::failure_redirect, }, }; -use video_iter::VideoFetchStream; +use video_iter::{FeedResultType, VideoFetchStream}; use video_loader::{BgView, VideoView}; use overlay::HomeButtonOverlay; @@ -171,6 +171,7 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { return; } f.start = 1; + f.limit = 15; }); video_queue.update_untracked(|v| { if v.len() > 1 { @@ -186,52 +187,47 @@ pub fn PostViewWithUpdates(initial_post: Option<PostDetails>) -> impl IntoView { let (nsfw_enabled, _, _) = use_local_storage::<bool, FromToStringCodec>(NSFW_TOGGLE_STORE); let auth_canisters: RwSignal<Option<Canisters<true>>> = expect_context(); - let fetch_video_action = create_action(move |_| { - async move { - loop { - let Some(cursor) = fetch_cursor.try_get_untracked() else { - return; - }; - let auth_canisters = auth_canisters.get_untracked(); - let nsfw_enabled = nsfw_enabled.get_untracked(); - let unauth_canisters = unauth_canisters(); + let fetch_video_action = create_action(move |_| async move { + loop { + let Some(cursor) = fetch_cursor.try_get_untracked() else { + return; + }; + let auth_canisters = auth_canisters.get_untracked(); + let nsfw_enabled = nsfw_enabled.get_untracked(); + let unauth_canisters = unauth_canisters(); - let chunks = if let Some(canisters) = auth_canisters.as_ref() { - let fetch_stream = VideoFetchStream::new(canisters, cursor); - fetch_stream - .fetch_post_uids_ml_feed_chunked( - 3, - nsfw_enabled, - video_queue.get_untracked(), - ) - .await // fetch_post_uids_ml_feed_chunked - } else { - let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); - fetch_stream - .fetch_post_uids_ml_feed_chunked( - 3, - nsfw_enabled, - video_queue.get_untracked(), - ) - .await // fetch_post_uids_chunked - }; + let chunks = if let Some(canisters) = auth_canisters.as_ref() { + let fetch_stream = VideoFetchStream::new(canisters, cursor); + fetch_stream + .fetch_post_uids_hybrid(3, nsfw_enabled, video_queue.get_untracked()) + .await + } else { + let fetch_stream = VideoFetchStream::new(&unauth_canisters, cursor); + fetch_stream + .fetch_post_uids_hybrid(3, nsfw_enabled, video_queue.get_untracked()) + .await + }; - let res = try_or_redirect!(chunks); - let mut chunks = res.posts_stream; - let mut cnt = 0; - while let Some(chunk) = chunks.next().await { - cnt += chunk.len(); - video_queue.try_update(|q| { - for uid in chunk { - let uid = try_or_redirect!(uid); - q.push(uid); - } - }); - } - if res.end || cnt >= 8 { - queue_end.try_set(res.end); - break; - } + let res = try_or_redirect!(chunks); + let mut chunks = res.posts_stream; + let mut cnt = 0; + while let Some(chunk) = chunks.next().await { + cnt += chunk.len(); + video_queue.try_update(|q| { + for uid in chunk { + let uid = try_or_redirect!(uid); + q.push(uid); + } + }); + } + leptos::logging::log!("feed type: {:?}", res.res_type); + if res.res_type == FeedResultType::PostCache { + fetch_cursor.try_update(|c| c.advance()); + } + + if res.end || cnt >= 8 { + queue_end.try_set(res.end); + break; } } }); diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index a8420ab1..562ccf4c 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -23,9 +23,16 @@ pub async fn post_liked_by_me( type PostsStream<'a> = Pin<Box<dyn Stream<Item = Vec<Result<PostDetails, PostViewError>>> + 'a>>; +#[derive(Debug, Eq, PartialEq)] +pub enum FeedResultType { + PostCache, + MLFeed, +} + pub struct FetchVideosRes<'a> { pub posts_stream: PostsStream<'a>, pub end: bool, + pub res_type: FeedResultType, } pub struct VideoFetchStream<'a, const AUTH: bool> { @@ -39,7 +46,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { } pub async fn fetch_post_uids_chunked( - self, + &self, chunks: usize, allow_nsfw: bool, ) -> Result<FetchVideosRes<'a>, PostViewError> { @@ -62,6 +69,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { return Ok(FetchVideosRes { posts_stream: Box::pin(futures::stream::empty()), end: true, + res_type: FeedResultType::PostCache, }) } post_cache::Result_::Err(_) => { @@ -82,11 +90,12 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, + res_type: FeedResultType::PostCache, }) } pub async fn fetch_post_uids_ml_feed_chunked( - self, + &self, chunks: usize, _allow_nsfw: bool, video_queue: Vec<PostDetails>, @@ -108,11 +117,10 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { let top_posts = match top_posts_fut.await { Ok(top_posts) => top_posts, Err(e) => { - leptos::logging::log!("error fetching posts: {:?}", e); - return Ok(FetchVideosRes { - posts_stream: Box::pin(futures::stream::empty()), - end: true, - }); + leptos::logging::log!("error fetching posts: {:?}", e); // TODO: to be removed + return Err(PostViewError::MLFeedError( + "ML feed server failed to send results".into(), + )); } }; @@ -127,6 +135,7 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { Ok(FetchVideosRes { posts_stream: Box::pin(chunk_stream), end, + res_type: FeedResultType::MLFeed, }) } @@ -135,7 +144,32 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { return Ok(FetchVideosRes { posts_stream: Box::pin(futures::stream::empty()), end: true, + res_type: FeedResultType::MLFeed, }); } } + + pub async fn fetch_post_uids_hybrid( + &self, + chunks: usize, + _allow_nsfw: bool, + video_queue: Vec<PostDetails>, + ) -> Result<FetchVideosRes<'a>, PostViewError> { + // If cursor.start is < 15, fetch from fetch_post_uids_chunked + // else fetch from fetch_post_uids_ml_feed_chunked + // if that fails fallback to fetch_post_uids_chunked + + if self.cursor.start < 15 { + return self.fetch_post_uids_chunked(chunks, _allow_nsfw).await; + } else { + let res = self + .fetch_post_uids_ml_feed_chunked(chunks, _allow_nsfw, video_queue) + .await; + + match res { + Ok(res) => return Ok(res), + Err(_) => return self.fetch_post_uids_chunked(chunks, _allow_nsfw).await, + } + } + } } diff --git a/ssr/src/page/root.rs b/ssr/src/page/root.rs index 0398357f..fa49e47f 100644 --- a/ssr/src/page/root.rs +++ b/ssr/src/page/root.rs @@ -35,33 +35,34 @@ async fn get_top_post_id() -> Result<Option<(Principal, u64)>, ServerFnError> { Ok(Some((top_item.publisher_canister_id, top_item.post_id))) } -#[server] -async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnError> { - use crate::utils::ml_feed::ml_feed_grpc::get_start_feed; +// TODO: Implement this when we shift to the new ml feed for first post +// #[server] +// async fn get_top_post_id_mlfeed() -> Result<Option<(Principal, u64)>, ServerFnError> { +// use crate::utils::ml_feed::ml_feed_grpc::get_start_feed; - let canisters = unauth_canisters(); - let user_canister_principal = canisters.user_canister(); - let top_posts_fut = get_start_feed(&user_canister_principal, 1, vec![]); +// let canisters = unauth_canisters(); +// let user_canister_principal = canisters.user_canister(); +// let top_posts_fut = get_start_feed(&user_canister_principal, 1, vec![]); - let top_items = match top_posts_fut.await { - Ok(top_posts) => top_posts, - Err(e) => { - log::error!("failed to fetch top post ml feed: {:?}", e); - return Err(ServerFnError::ServerError( - "failed to fetch top post ml feed".to_string(), - )); - } - }; - let Some(top_item) = top_items.first() else { - return Ok(None); - }; +// let top_items = match top_posts_fut.await { +// Ok(top_posts) => top_posts, +// Err(e) => { +// log::error!("failed to fetch top post ml feed: {:?}", e); +// return Err(ServerFnError::ServerError( +// "failed to fetch top post ml feed".to_string(), +// )); +// } +// }; +// let Some(top_item) = top_items.first() else { +// return Ok(None); +// }; - Ok(Some((top_item.0, top_item.1))) -} +// Ok(Some((top_item.0, top_item.1))) +// } #[component] pub fn RootPage() -> impl IntoView { - let target_post = create_resource(|| (), |_| get_top_post_id_mlfeed()); + let target_post = create_resource(|| (), |_| get_top_post_id()); view! { <Suspense fallback=FullScreenSpinner> diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 81bc59d0..05cef04c 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -2,10 +2,11 @@ use candid::Principal; use serde::{Deserialize, Serialize}; use crate::{ - canister::individual_user_template::PostDetailsForFrontend, state::canisters::Canisters, + canister::individual_user_template::{PostDetailsForFrontend, PostStatus as PostStatusCandid}, + state::canisters::Canisters, }; -use super::profile::propic_from_principal; +use super::{profile::propic_from_principal, types::PostStatus}; use ic_agent::AgentError; use thiserror::Error; @@ -18,6 +19,8 @@ pub enum PostViewError { Canister(String), #[error("http fetch error {0}")] HttpFetch(#[from] reqwest::Error), + #[error("ml feed error {0}")] + MLFeedError(String), } #[derive(Clone, Copy, PartialEq, Debug)] @@ -30,7 +33,7 @@ impl Default for FetchCursor { fn default() -> Self { Self { start: 0, - limit: 25, + limit: 15, } } } @@ -38,7 +41,7 @@ impl Default for FetchCursor { impl FetchCursor { pub fn advance(&mut self) { self.start += self.limit; - self.limit = 25; + self.limit = 30; } } @@ -112,10 +115,10 @@ pub async fn get_post_uid<const AUTH: bool>( } }; - // TODO: Add this filter in new method - // if post_details.status == PostStatus::BannedDueToUserReporting { - // return Ok(None); - // } + // TODO: temporary patch in frontend to not show banned videos, to be removed later after NSFW tagging + if PostStatus::from(&post_details.status) == PostStatus::BannedDueToUserReporting { + return Ok(None); + } let post_uuid = &post_details.video_uid; let req_url = format!( diff --git a/ssr/src/utils/types.rs b/ssr/src/utils/types.rs index e38f2ff9..5d4a7fe4 100644 --- a/ssr/src/utils/types.rs +++ b/ssr/src/utils/types.rs @@ -1,3 +1,29 @@ +use crate::canister::individual_user_template::PostStatus as PostStatusCandid; use candid::Principal; pub type PostId = (Principal, u64); + +#[derive(PartialEq, Debug, Eq)] +pub enum PostStatus { + BannedForExplicitness, + BannedDueToUserReporting, + Uploaded, + CheckingExplicitness, + ReadyToView, + Transcoding, + Deleted, +} + +impl From<&PostStatusCandid> for PostStatus { + fn from(status: &PostStatusCandid) -> Self { + match status { + PostStatusCandid::BannedForExplicitness => PostStatus::BannedForExplicitness, + PostStatusCandid::BannedDueToUserReporting => PostStatus::BannedDueToUserReporting, + PostStatusCandid::Uploaded => PostStatus::Uploaded, + PostStatusCandid::CheckingExplicitness => PostStatus::CheckingExplicitness, + PostStatusCandid::ReadyToView => PostStatus::ReadyToView, + PostStatusCandid::Transcoding => PostStatus::Transcoding, + PostStatusCandid::Deleted => PostStatus::Deleted, + } + } +} From 5013938bf58b3469858a6ec5a7c0d927d0ef2adb Mon Sep 17 00:00:00 2001 From: komal-rs <komalsai@gobazzinga.io> Date: Thu, 8 Aug 2024 22:06:23 +0530 Subject: [PATCH 9/9] clippy fix --- ssr/src/page/post_view/video_iter.rs | 6 +++--- ssr/src/utils/posts.rs | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ssr/src/page/post_view/video_iter.rs b/ssr/src/page/post_view/video_iter.rs index 562ccf4c..5f603864 100644 --- a/ssr/src/page/post_view/video_iter.rs +++ b/ssr/src/page/post_view/video_iter.rs @@ -160,15 +160,15 @@ impl<'a, const AUTH: bool> VideoFetchStream<'a, AUTH> { // if that fails fallback to fetch_post_uids_chunked if self.cursor.start < 15 { - return self.fetch_post_uids_chunked(chunks, _allow_nsfw).await; + self.fetch_post_uids_chunked(chunks, _allow_nsfw).await } else { let res = self .fetch_post_uids_ml_feed_chunked(chunks, _allow_nsfw, video_queue) .await; match res { - Ok(res) => return Ok(res), - Err(_) => return self.fetch_post_uids_chunked(chunks, _allow_nsfw).await, + Ok(res) => Ok(res), + Err(_) => self.fetch_post_uids_chunked(chunks, _allow_nsfw).await, } } } diff --git a/ssr/src/utils/posts.rs b/ssr/src/utils/posts.rs index 05cef04c..cca2125a 100644 --- a/ssr/src/utils/posts.rs +++ b/ssr/src/utils/posts.rs @@ -2,8 +2,7 @@ use candid::Principal; use serde::{Deserialize, Serialize}; use crate::{ - canister::individual_user_template::{PostDetailsForFrontend, PostStatus as PostStatusCandid}, - state::canisters::Canisters, + canister::individual_user_template::PostDetailsForFrontend, state::canisters::Canisters, }; use super::{profile::propic_from_principal, types::PostStatus};