Status

daemon/src/api/indexing_status.rs

@@ -0,0 +1,13 @@
+use super::*;
+
+pub(super) async fn indexing_status(index: DocumentIndex) -> Result<impl warp::Reply, Infallible> {
+    let status = index.status().await;
+    let status_json = match serde_json::to_string(&status) {
+        Ok(json) => json,
+        Err(e) => {
+            error!("Failed to serialize indexing status: {}", e);
+            return Ok(Response::builder().status(500).body("Failed to serialize indexing status".to_string()).unwrap());
+        }
+    };
+    Ok(Response::builder().header("Content-Type", "application/json").body(status_json).unwrap())
+}

daemon/src/api/mod.rs

@@ -4,13 +4,15 @@ use warp::{Filter, http::Response};
 use std::{convert::Infallible, net::SocketAddr};
 
 mod bodies;
+mod indexing_status;
 mod local_search;
 mod search;
 mod results;
 mod version;
 use {
-    local_search::*,
     bodies::*,
+    indexing_status::*,
+    local_search::*,
     search::*,
     results::*,
     version::*,
@@ -76,6 +78,12 @@ impl SearchPark {
 pub async fn serve_api(config: Arc<Args>, index: DocumentIndex, search_park: Arc<SearchPark>, kamilata: NodeController) {
     let hello_world = warp::path::end().map(|| "Hello, World at root!");
 
+    let index2 = index.clone();
+    let indexing_status = warp::get()
+        .and(warp::path("indexing-status"))
+        .map(move || index2.clone())
+        .and_then(indexing_status);
+
     let local_search = warp::get()
         .and(warp::path("local-search"))
         .and(warp::query::<ApiSearchQuery>())
@@ -125,6 +133,7 @@ pub async fn serve_api(config: Arc<Args>, index: DocumentIndex, search_park: Arc
 
     let routes = warp::any().and(
         hello_world
+            .or(indexing_status)
             .or(local_search)
             .or(search)
             .or(results)

daemon/src/dns_pins.rs

@@ -13,6 +13,7 @@ pub async fn manage_dns_pins(config: Arc<Args>) {
         dns_pins_interval = 60*3;
     }
 
+    // TODO: Make use of named pins for DNS pins
     // Find old pins and look for the previous DNS pins
     let old_pins = match list_pinned(&config.ipfs_rpc).await {
         Ok(pins) => pins,

daemon/src/index/index.rs

@@ -3,6 +3,7 @@ use super::*;
 #[derive(Clone)]
 pub struct DocumentIndex {
     config: Arc<Args>,
+    status: Arc<RwLock<IndexingStatus>>,
     inner: Arc<RwLock<DocumentIndexInner>>,
 }
 
@@ -11,6 +12,7 @@ impl DocumentIndex {
     pub async fn new(config: Arc<Args>) -> DocumentIndex {
         DocumentIndex {
             inner: Arc::new(RwLock::new(DocumentIndexInner::new(Arc::clone(&config)).await)),
+            status: Arc::new(RwLock::new(IndexingStatus::default())),
             config,
         }
     }
@@ -48,7 +50,7 @@ impl DocumentIndex {
             let mut to_list = Vec::new();
             let mut to_load = HashMap::new();
             let mut to_load_unprioritized = HashMap::new();
-
+            
             // List pinned elements
             let pinned = match list_pinned(&self.config.ipfs_rpc).await {
                 Ok(pinned) => pinned,
@@ -64,13 +66,15 @@ impl DocumentIndex {
             };
             last_printed_error = None;
             to_list.extend(pinned.iter().filter_map(normalize_cid).filter(|cid| !listed.contains(cid)));
+            self.set_status(listed.len(), to_list.len(), loaded.len(), to_load.len(), to_load_unprioritized.len()).await;
 
             // Explore directories
             let start = Instant::now();
-            let mut i = 0;
             if !to_list.is_empty() {debug!("{} elements to list", to_list.len())}
             while let Some(cid) = to_list.pop() {
                 if !listed.insert(cid.clone()) {continue}
+                self.set_status(listed.len(), to_list.len()+1, loaded.len(), to_load.len(), to_load_unprioritized.len()).await;
+
                 let new_links = match ls(ipfs_rpc, cid.clone()).await {
                     Ok(new_links) => new_links,
                     Err(e) => {
@@ -94,29 +98,29 @@ impl DocumentIndex {
                 }
                 to_list.sort();
                 to_list.dedup();
-                i += 1;
-                if i % 500 == 0 {
-                    debug!("Still listing pinned files ({i} in {:.02})", start.elapsed().as_secs_f32());
-                }
             }
 
             // Load documents
-            i = 0;
+            to_load_unprioritized.retain(|cid, _| !to_load.contains_key(cid));
             if !to_load.is_empty() {debug!("{} documents to load ({:.02?}s)", to_load.len(), start.elapsed().as_secs_f32())}
-            for (cid, (name, parent_cid)) in to_load.drain().chain(to_load_unprioritized.drain()) {
-                if !loaded.insert(cid.clone()) {continue}
+            let (to_load_len, to_load_unprioritized_len) = (to_load.len(), to_load_unprioritized.len());
+            for (i, (cid, (name, parent_cid))) in to_load.drain().chain(to_load_unprioritized.drain()).enumerate() {
+                let remaining_to_load = to_load_len.saturating_sub(i);
+                let remaining_unprioritized = std::cmp::min(to_load_unprioritized_len, to_load_len + to_load_unprioritized_len - i);
+                self.set_status(listed.len(), to_list.len(), loaded.len(), remaining_to_load, remaining_unprioritized).await;
+
+                loaded.insert(cid.clone());
                 let Ok(document) = fetch_document(ipfs_rpc, &cid).await else {continue};
                 let Some(inspected) = inspect_document(document) else {continue};
                 self.add_document(&cid, inspected).await;
                 self.add_ancestor(&cid, name, false, &parent_cid).await;
-                i += 1;
-                if i % 500 == 0 {
-                    debug!("Still loading files ({i} in {:.02})", start.elapsed().as_secs_f32());
-                }
             }
 
             // Update filter
+            self.set_status(listed.len(), 0, loaded.len(), 0, 0).await;
+            self.set_status_updating_filter(true).await;
             self.update_filter().await;
+            self.set_status_updating_filter(false).await;
             let load = self.get_filter().await.load()*100.0;
             if load != previous_load {
                 previous_load = load;
@@ -126,6 +130,24 @@ impl DocumentIndex {
         }
     }
 
+    async fn set_status(&self, listed: usize, to_list: usize, loaded: usize, to_load: usize, to_load_unprioritized: usize) {
+        let mut status = self.status.write().await;
+        status.listed = listed;
+        status.to_list = to_list;
+        status.loaded = loaded;
+        status.to_load = to_load;
+        status.to_load_unprioritized = to_load_unprioritized;
+    }
+
+    async fn set_status_updating_filter(&self, updating_filter: bool) {
+        let mut status = self.status.write().await;
+        status.updating_filter = updating_filter;
+    }
+
+    pub async fn status(&self) -> IndexingStatus {
+        self.status.read().await.clone()
+    }
+
     pub async fn documents(&self) -> HashSet<String> {
         self.inner.read().await.documents()
     }

daemon/src/index/inner_im.rs

@@ -33,6 +33,7 @@ impl DocumentIndexInner {
         }
     }   
 
+    #[allow(dead_code)]
     pub(super) async fn sweep(&mut self) {}
 
     pub fn documents(&self) -> HashSet<String> {

daemon/src/index/mod.rs

@@ -8,8 +8,10 @@ const REFRESH_INTERVAL: u64 = 100;
 const SWEEP_INTERVAL: u64 = 30;
 
 mod index;
+mod status;
 mod inner_common;
 pub use index::*;
+pub use status::*;
 
 #[cfg(any(feature = "database-lmdb", feature = "database-mdbx"))]
 mod inner_db;

daemon/src/index/status.rs

@@ -0,0 +1,11 @@
+use serde::{Serialize, Deserialize};
+
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
+pub struct IndexingStatus {
+    pub listed: usize,
+    pub to_list: usize,
+    pub loaded: usize,
+    pub to_load: usize,
+    pub to_load_unprioritized: usize,
+    pub updating_filter: bool,
+}

daemon/src/main.rs

@@ -1,4 +1,5 @@
 #![allow(clippy::module_inception)]
+#![allow(clippy::map_clone)] /* This lint doesn't make the difference between Arc clones and regular clones */
 #![recursion_limit = "256"]
 
 mod result;

webui/index.html

@@ -12,6 +12,7 @@
     <link data-trunk rel="css" href="src/pages/results/results.css" />
     <link data-trunk rel="css" href="src/components/search_bar/search_bar.css" />
     <link data-trunk rel="css" href="src/components/connection_status/connection_status.css" />
+    <link data-trunk rel="css" href="src/components/indexing_status/indexing_status.css" />
     <link data-trunk rel="css" href="src/components/result/result.css" />
     <link data-trunk rel="copy-dir" href="assets" />
     <meta property="og:site_name" content="Admarus" />
@@ -48,11 +49,6 @@ <h1>Admarus</h1>
                 </div>
             </div>
         </section>
-    </main>    
-    <script>
-        // https://css-tricks.com/the-trick-to-viewport-units-on-mobile/
-        let vh = window.innerHeight * 0.01;
-        document.documentElement.style.setProperty('--vh', `${vh}px`);
-    </script>
+    </main>
 </body>
 </html>

webui/src/api.rs

@@ -13,14 +13,13 @@ pub enum ApiError {
 
 impl ApiError {
     pub fn to_format_parts(&self) -> (&'static str, Vec<String>, String) {
-        let (title, recommandations, details) = match self {
+        let (title, recommandations) = match self {
             ApiError::InputJson(e) => (
                 "Failed to craft request",
                 vec![
                     String::from("Open an issue on GitHub"),
                     String::from("Try again"),
-                ],
-                format!("InputJson: {e}")
+                ]
             ),
             ApiError::OutputJson(e) => (
                 "Failed to read results",
@@ -29,8 +28,7 @@ impl ApiError {
                     String::from("Make sure the daemon address is correct"),
                     String::from("Open an issue on GitHub"),
                     String::from("Try again"),
-                ],
-                format!("OutputJson: {e}")
+                ]
             ),
             ApiError::Fetch(e) => (
                 "Failed to send query",
@@ -39,17 +37,15 @@ impl ApiError {
                     String::from("Make sure the daemon address is correct"),
                     String::from("Make sure CORS is properly configured"),
                     String::from("Try again"),
-                ],
-                format!("Fetch: {}", e.clone().dyn_into::<js_sys::Error>().unwrap().message())
+                ]
             ),
             ApiError::NotText(e) => (
                 "Invalid response",
                 vec![
                     String::from("Make sure the daemon address is correct"),
                     String::from("Open an issue on GitHub"),
                     String::from("Try again"),
-                ],
-                format!("NotText: {}", e.clone().dyn_into::<js_sys::Error>().unwrap().message())
+                ]
             ),
             ApiError::BadRequest(e) => (
                 "Failed to communicate with daemon",
@@ -58,28 +54,39 @@ impl ApiError {
                     String::from("Make sure the daemon address is correct"),
                     String::from("Open an issue on GitHub"),
                     String::from("Try again"),
-                ],
-                format!("BadRequest: {e}")
+                ]
             ),
             ApiError::Server(e) => (
                 "Daemon is having issues",
                 vec![
                     String::from("Make sure the daemon is up to date"),
                     String::from("Open an issue on GitHub"),
                     String::from("Try again"),
-                ],
-                format!("Server: {e}")
+                ]
             ),
             ApiError::Unknown(e) => (
                 "Unknown error",
                 vec![
                     String::from("Make sure the daemon address is correct"),
                     String::from("Try again"),
-                ],
-                format!("Unknown: {e}")
+                ]
             ),
         };
-        (title, recommandations, details)
+        (title, recommandations, self.to_string())
+    }
+}
+
+impl std::fmt::Display for ApiError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            ApiError::InputJson(e) => write!(f, "InputJson: {}", e),
+            ApiError::OutputJson(e) => write!(f, "OutputJson: {}", e),
+            ApiError::Fetch(e) => write!(f, "Fetch: {}", e.clone().dyn_into::<js_sys::Error>().unwrap().message()),
+            ApiError::NotText(e) => write!(f, "NotText: {}", e.clone().dyn_into::<js_sys::Error>().unwrap().message()),
+            ApiError::BadRequest(e) => write!(f, "BadRequest: {}", e),
+            ApiError::Server(e) => write!(f, "Server: {}", e),
+            ApiError::Unknown(e) => write!(f, "Unknown: {}", e),
+        }
     }
 }
 
@@ -89,7 +96,7 @@ impl From<serde_json::Error> for ApiError {
     }
 }
 
-async fn get<T: DeserializeOwned>(url: impl AsRef<str>) -> Result<T, ApiError> {
+pub async fn get<T: DeserializeOwned>(url: impl AsRef<str>) -> Result<T, ApiError> {
     api_custom_method(url, "GET", ()).await
 }
 

webui/src/components/indexing_status/api.rs

@@ -0,0 +1,5 @@
+use crate::prelude::*;
+
+pub async fn get_indexing_status(rpc_addr: &str) -> Result<IndexingStatus, ApiError> {
+    get(format!("{rpc_addr}/indexing-status")).await
+}

webui/src/components/indexing_status/indexing_status.css

@@ -0,0 +1,28 @@
+#indexing-status {
+    box-sizing: border-box;
+    width: 100%;
+    padding: 2rem 1rem 0 1rem;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+}
+
+#indexing-status-message {
+    margin: 0 0 .25rem 0;
+}
+
+#indexing-status-progress {
+    width: min(100%, 20rem);
+    height: 1rem;
+    border-radius: .5rem;
+    overflow: hidden;
+    background-color: #eaeaea;
+    /* border: 2px solid #ccc; */
+    border: 2px solid #2ecc71;
+}
+
+#indexing-status-inner {
+    background-color: #2ecc71;
+    height: 100%;
+    transition: width 1s linear;
+}

webui/src/components/indexing_status/indexing_status.html

@@ -0,0 +1,22 @@
+<div id="indexing-status">
+    <virtual present-if={{exploring}}>
+        <p id="indexing-status-message">Your daemon is discovering files ({{progress_value}} / {{progress_max}}).</p>
+        <div id="indexing-status-progress">
+            <div id="indexing-status-inner" style="width: calc(100% * {{progress_value}} / {{progress_max}});"></div>
+        </div>
+    </virtual>
+
+    <virtual present-if={{indexing}}>
+        <p id="indexing-status-message">Your daemon is indexing files ({{progress_value}} / {{progress_max}}).</p>
+        <div id="indexing-status-progress">
+            <div id="indexing-status-inner" style="width: calc(100% * {{progress_value}} / {{progress_max}});"></div>
+        </div>
+    </virtual>
+
+    <virtual present-if={{building_filter}}>
+        <p id="indexing-status-message">Your daemon is compiling its index.</p>
+        <div id="indexing-status-progress">
+            <div id="indexing-status-inner" style="width: 100%;"></div>
+        </div>
+    </virtual>
+</div>