Log cache contents every hour

Cargo.toml

@@ -45,7 +45,7 @@ smartstring = "1.0.1"
 tera = { version = "1.20.0", default-features = false }
 thiserror = "1.0.63"
 time = { version = "0.3.36", features = ["formatting", "macros", "parsing", "serde"] }
-tokio = { version = "1.39.2", features = ["macros", "net", "rt-multi-thread"] }
+tokio = { version = "1.39.2", features = ["macros", "net", "rt-multi-thread", "time"] }
 tower = "0.4.13"
 tower-http = { version = "0.5.2", features = ["set-header", "trace"] }
 tracing = "0.1.40"
@@ -199,7 +199,7 @@ ignored_unit_patterns = "deny"
 impl_trait_in_params = "deny"
 implicit_clone = "deny"
 imprecise_flops = "deny"
-infinite_loop = "deny"
+#infinite_loop = "deny"
 index_refutable_slice = "deny"
 invalid_upcast_comparisons = "deny"
 items_after_statements = "deny"

src/consts.rs

@@ -1,4 +1,5 @@
 //! Constants and program-wide compile-time configuration
+use std::time::Duration;
 use time::{format_description::FormatItem, macros::format_description};
 
 /// The "User-Agent" value sent in outgoing HTTP requests
@@ -57,6 +58,9 @@ pub(crate) static HTML_TIMESTAMP_FORMAT: &[FormatItem<'_>] =
 /// This list must be kept in sorted order; this is enforced by a test below.
 pub(crate) static FAST_NOT_EXIST: &[&str] = &[".bzr", ".git", ".nols", ".svn"];
 
+/// Interval between periodic logging of the Zarr manifest cache's contents
+pub(crate) const ZARR_MANIFEST_CACHE_DUMP_PERIOD: Duration = Duration::from_secs(3600);
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/main.rs

@@ -9,7 +9,9 @@ mod paths;
 mod s3;
 mod streamutil;
 mod zarrman;
-use crate::consts::{CSS_CONTENT_TYPE, DEFAULT_API_URL, SERVER_VALUE};
+use crate::consts::{
+    CSS_CONTENT_TYPE, DEFAULT_API_URL, SERVER_VALUE, ZARR_MANIFEST_CACHE_DUMP_PERIOD,
+};
 use crate::dandi::DandiClient;
 use crate::dav::{DandiDav, Templater};
 use crate::zarrman::{ManifestFetcher, ZarrManClient};
@@ -97,6 +99,7 @@ async fn run() -> anyhow::Result<()> {
     let args = Arguments::parse();
     let dandi = DandiClient::new(args.api_url)?;
     let zarrfetcher = ManifestFetcher::new()?;
+    zarrfetcher.install_periodic_dump(ZARR_MANIFEST_CACHE_DUMP_PERIOD);
     let zarrman = ZarrManClient::new(zarrfetcher);
     let templater = Templater::new(args.title)?;
     let dav = Arc::new(DandiDav {

src/zarrman/fetcher.rs

@@ -9,7 +9,9 @@ use moka::{
     future::{Cache, CacheBuilder},
     ops::compute::{CompResult, Op},
 };
+use serde::Serialize;
 use std::sync::Arc;
+use std::time::Duration;
 use url::Url;
 
 /// A client for fetching & caching data from the manifest tree
@@ -134,4 +136,54 @@ impl ManifestFetcher {
         };
         Ok(entry.into_value())
     }
+
+    pub(crate) fn install_periodic_dump(&self, period: Duration) {
+        let this = self.clone();
+        let mut schedule = tokio::time::interval(period);
+        schedule.reset(); // Don't tick immediately
+        schedule.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+        tokio::spawn({
+            async move {
+                loop {
+                    schedule.tick().await;
+                    this.log_cache();
+                }
+            }
+        });
+    }
+
+    pub(crate) fn log_cache(&self) {
+        let entries = self
+            .cache
+            .iter()
+            .map(|(path, manifest)| EntryStat {
+                manifest_path: path.to_string(),
+                size: manifest.get_size(),
+            })
+            .collect::<Vec<_>>();
+        match serde_json::to_string(&entries) {
+            Ok(entries_json) => {
+                tracing::debug!(
+                    cache_event = "dump",
+                    cache = "zarr-manifests",
+                    %entries_json,
+                    "Dumping cached manifests and their sizes",
+                );
+            }
+            Err(e) => {
+                tracing::warn!(
+                    cache_event = "dump-error",
+                    cache = "zarr-manifests",
+                    error = %e,
+                    "Failed to serialize cache contents as JSON",
+                );
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+struct EntryStat {
+    manifest_path: String,
+    size: usize,
 }

tools/zarr-cache-stats.py

@@ -248,6 +248,8 @@ def process_logs(logfiles: Iterable[Path]) -> Events:
                             )
                         )
                         last_accesses.pop(manifest_path, None)
+                    case "dump" | "dump-error":
+                        pass
                     case other:
                         log.warning(
                             "Invalid 'cache_event' field value %r: %s", other, lg.line