writing videos to local data and (hardcoded) reading a frame and pres…

…enting it in timeline

src-tauri/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2021"
 tauri-build = { version = "1.5", features = [] }
 
 [dependencies]
-tauri = { version = "1.5", features = [ "system-tray", "shell-open"] }
+tauri = { version = "1.5", features = ["system-tray", "shell-open"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 
@@ -43,6 +43,12 @@ candle-nn = "0.3.2"
 candle-transformers = "0.3.2"
 tokenizers = "0.15.0"
 lazy_static = "1.4.0"
+base64 = "0.21.7"
+
+# Server
+axum = "0.7.4"
+tokio = { version = "1", features = ["full"] }
+hyper = "1.1"
 
 [profile.release]
 opt-level = 3

src-tauri/build.rs

@@ -1,3 +1,17 @@
+use std::env;
+use std::path::PathBuf;
+
 fn main() {
+    // // Get the manifest directory (location of Cargo.toml)
+    // let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+    //
+    // // Construct the path to the ffmpeg binary
+    // let ffmpeg_binary_path = PathBuf::from(manifest_dir)
+    //     .join("binaries")
+    //     .join("ffmpeg");
+    //
+    // // Convert the path to a string and add it to the linker search path
+    // let ffmpeg_binary_path = ffmpeg_binary_path.to_str().unwrap();
+    // println!("cargo:rustc-link-search=native={}", ffmpeg_binary_path);
     tauri_build::build()
 }

src-tauri/src/core/core.rs

@@ -41,7 +41,7 @@ impl CaptureHandles {
     }
 }
 
-pub fn start_recording() -> CaptureHandles {
+pub fn start_recording(local_data_dir: String) -> CaptureHandles {
     let config_path = "models/gte-small/config.json";
     let tokenizer_path = "models/gte-small/tokenizer.json";
     let weights_path = "models/gte-small/model.safetensors";
@@ -57,11 +57,20 @@ pub fn start_recording() -> CaptureHandles {
     // Capture thread
     let buffer_clone = frame_buffer.clone();
 
+    let local_data_dir_capture_handle = local_data_dir.clone();
+
     let capture_handle = thread::spawn(move || {
-        capture_screenshots(buffer_clone, &ocr_pool, control_receiver)
-            .expect("Error capturing screenshots");
+        capture_screenshots(
+            buffer_clone,
+            &ocr_pool,
+            control_receiver,
+            local_data_dir_capture_handle,
+        )
+        .expect("Error capturing screenshots");
     });
 
+    let local_data_dir_stream_handle = local_data_dir.clone();
+
     let stream_handle = thread::spawn(move || {
         // Main thread for processing frames
         let (buffer, cvar) = &*frame_buffer;
@@ -73,7 +82,7 @@ pub fn start_recording() -> CaptureHandles {
 
             // Drain frames and process with FFmpeg
             let frames_to_process = frames.drain(..).collect::<Vec<_>>();
-            stream_to_ffmpeg(frames_to_process);
+            stream_to_ffmpeg(frames_to_process, local_data_dir_stream_handle.clone());
         }
     });
 
@@ -88,10 +97,12 @@ fn capture_screenshots(
     frame_buffer: Arc<(Mutex<Vec<DynamicImage>>, Condvar)>,
     ocr_pool: &ThreadPool,
     control_receiver: mpsc::Receiver<ControlMessage>,
+    local_data_dir: String,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let screens = Screen::all()?;
     let screen = screens.first().unwrap();
     let mut is_paused = false;
+    let local_data_dir_clone = local_data_dir.clone();
 
     loop {
         // Check for control messages
@@ -101,7 +112,7 @@ fn capture_screenshots(
                 ControlMessage::Resume => is_paused = false,
                 ControlMessage::Stop => {
                     // Process the frames
-                    process_remaining_frames(&frame_buffer);
+                    process_remaining_frames(&frame_buffer, local_data_dir_clone);
                     return Ok(());
                 }
             }
@@ -159,11 +170,12 @@ fn perform_ocr(dynamic_image: &DynamicImage) -> Result<String, Box<dyn std::erro
     Ok(text)
 }
 
-fn stream_to_ffmpeg(frames: Vec<DynamicImage>) {
+fn stream_to_ffmpeg(frames: Vec<DynamicImage>, local_data_dir: String) {
     let encode_pool = ThreadPool::new(IMAGE_ENCODE_THREADS); // Define NUM_ENCODE_THREADS based on your CPU
     print!("getting ready to stream..");
     let time = Utc::now();
-    let output_name = format!("{}.mp4", time);
+    let local_data_dir_clone = local_data_dir.clone();
+    let output_name = format!("{}/{}.mp4", local_data_dir_clone, time);
     let mut child = Command::new("ffmpeg")
         .args([
             "-f",
@@ -230,12 +242,16 @@ fn stream_to_ffmpeg(frames: Vec<DynamicImage>) {
     println!("waited?");
 }
 
-fn process_remaining_frames(frame_buffer: &Arc<(Mutex<Vec<DynamicImage>>, Condvar)>) {
+fn process_remaining_frames(
+    frame_buffer: &Arc<(Mutex<Vec<DynamicImage>>, Condvar)>,
+    local_data_dir: String,
+) {
+    let local_data_dir_clone = local_data_dir.clone();
     let (lock, _) = &**frame_buffer;
     let mut frames = lock.lock().unwrap();
 
     if !frames.is_empty() {
         let frames_to_process = frames.drain(..).collect::<Vec<_>>();
-        stream_to_ffmpeg(frames_to_process);
+        stream_to_ffmpeg(frames_to_process, local_data_dir_clone);
     }
 }

src-tauri/src/core/mod.rs

@@ -1,5 +1,7 @@
 mod core;
 mod embed;
+mod video;
 
 pub use core::start_recording;
 pub use core::CaptureHandles;
+pub use video::extract_frames_from_video;

src-tauri/src/core/video.rs

@@ -1,9 +1,9 @@
 use ffmpeg_next::{
-    format, format::Pixel, frame, media, software::scaling, util::frame::video::Video,
+    format, format::Pixel, media, software::scaling, util::frame::video::Video,
 };
 use image::{DynamicImage, ImageBuffer, Rgb};
 
-fn extract_frames_from_video(
+pub fn extract_frames_from_video(
     video_path: &str,
     frame_numbers: &[i64],
 ) -> Result<Vec<DynamicImage>, ffmpeg_next::Error> {
@@ -21,11 +21,12 @@ fn extract_frames_from_video(
     let context_decoder =
         ffmpeg_next::codec::context::Context::from_parameters(input_stream.parameters())?;
     let mut decoder = context_decoder.decoder().video()?;
+    println!("Trying to scale...");
     let mut scaler = scaling::Context::get(
         decoder.format(),
         decoder.width(),
         decoder.height(),
-        Pixel::RGB8,
+        Pixel::RGB24,
         decoder.width(),
         decoder.height(),
         scaling::Flags::BILINEAR,
@@ -35,11 +36,11 @@ fn extract_frames_from_video(
         |decoder: &mut ffmpeg_next::decoder::Video| -> Result<Video, ffmpeg_next::Error> {
             let mut fni: usize = 0;
 
-            let mut decoded = frame::Video::empty();
+            let mut decoded = Video::empty();
             let mut vi: i64 = 0;
             if let Some(&last_frame) = frame_numbers.last() {
                 let mut rgb_frame = Video::empty();
-                while vi < last_frame && decoder.receive_frame(&mut decoded).is_ok() {
+                while vi <= last_frame && decoder.receive_frame(&mut decoded).is_ok() {
                     if vi == frame_numbers[fni] {
                         scaler.run(&decoded, &mut rgb_frame)?;
                         // next frame in list...
@@ -58,7 +59,7 @@ fn extract_frames_from_video(
             let frames = receive_and_process_frames(&mut decoder)?;
             for (i, _) in frame_numbers.iter().enumerate() {
                 let frame = frames.data(i);
-                let img: ImageBuffer<Rgb<u8>, Vec<u8>> = image::ImageBuffer::from_raw(
+                let img: ImageBuffer<Rgb<u8>, Vec<u8>> = ImageBuffer::from_raw(
                     decoder.width() as u32,
                     decoder.height() as u32,
                     frame.to_vec(),

src-tauri/src/main.rs

@@ -2,21 +2,35 @@
 #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
 
 use std::{
+    fs,
     sync::{Arc, Mutex},
 };
 use tauri::{
-    CustomMenuItem, LogicalPosition, Manager, SystemTray, SystemTrayEvent,
-    SystemTrayMenu, SystemTrayMenuItem,
+    CustomMenuItem, LogicalPosition, Manager, SystemTray, SystemTrayEvent, SystemTrayMenu,
+    SystemTrayMenuItem,
 };
+use tokio::sync::oneshot;
+use core::{start_recording, CaptureHandles};
 
 mod core;
+mod server;
 
-use core::{start_recording, CaptureHandles};
 
-// Learn more about Tauri commands at https://tauri.app/v1/guides/features/command
-#[tauri::command]
-fn greet(name: &str) -> String {
-    format!("Hello, {}! You've been greeted from Rust!", name)
+fn start_server(app_handle: tauri::AppHandle) {
+    println!("starting server...");
+    let local_data_dir = app_handle.path_resolver().app_local_data_dir();
+    let (tx, rx) = oneshot::channel();
+
+    if let Some(dir) = local_data_dir.clone() {
+        let path = dir.to_string_lossy().to_string();
+        if let Ok(()) = fs::create_dir_all(path.clone()) {
+            let path_clone = path.clone();
+            tokio::spawn(async move {
+                server::start_frame_server(tx, path_clone).await;
+            });
+        }
+    }
+    println!("started server...");
 }
 
 fn make_tray() -> SystemTray {
@@ -34,11 +48,20 @@ fn make_tray() -> SystemTray {
     return tray;
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
+    println!("starting app...");
+    // Wait for the server to start
+    // let _ = rx.await;
+
     let is_capturing = Arc::new(Mutex::new(false));
     let handles: Arc<Mutex<Option<CaptureHandles>>> = Arc::new(Mutex::new(None));
 
     tauri::Builder::default()
+        .setup(|app| {
+            start_server(app.app_handle());
+            Ok(())
+        })
         .on_window_event(|event| match event.event() {
             tauri::WindowEvent::CloseRequested { api, .. } => {
                 event.window().hide().unwrap();
@@ -67,16 +90,21 @@ fn main() {
                             std::process::exit(0);
                         }
                         "toggle_recording" => {
-                            if *is_capturing {
-                                item_handle.set_title("Start Recording").unwrap();
-                                if let Some(ref mut handles) = *handles {
-                                    handles.stop_recording()
+                            let local_data_dir = app.path_resolver().app_local_data_dir();
+
+                            if let Some(dir) = local_data_dir.clone() {
+                                if *is_capturing {
+                                    item_handle.set_title("Start Recording").unwrap();
+                                    if let Some(ref mut handles) = *handles {
+                                        handles.stop_recording()
+                                    }
+                                    *is_capturing = false;
+                                } else {
+                                    item_handle.set_title("Stop Recording").unwrap();
+                                    let path = dir.to_string_lossy().to_string();
+                                    *handles = Some(start_recording(path));
+                                    *is_capturing = true;
                                 }
-                                *is_capturing = false;
-                            } else {
-                                item_handle.set_title("Stop Recording").unwrap();
-                                *handles = Some(start_recording());
-                                *is_capturing = true;
                             }
                         }
                         "toggle_search" => {
@@ -109,7 +137,6 @@ fn main() {
                 _ => {}
             }
         })
-        .invoke_handler(tauri::generate_handler![greet])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");
 }

src/routes/timeline/+page.svelte

@@ -1 +1,64 @@
-<h1>Timeline</h1>
+<script>
+    import { onMount } from 'svelte';
+    import { pan } from 'svelte-gestures';
+
+    let frameNumber = 0;
+    let swipePosition = 0;
+    let imageSrc = '';
+    let debounceTimer;
+
+    function debounce(func, delay) {
+        return function(...args) {
+            clearTimeout(debounceTimer);
+            debounceTimer = setTimeout(() => func.apply(this, args), delay);
+        };
+    }
+
+    const loadImage = debounce(() => {
+        imageSrc = `http://localhost:3030/get_frame/${frameNumber}`;
+        console.log(imageSrc);
+    }, 100);
+
+    function updateFrame() {
+        const newFrame = Math.round(swipePosition);
+        if (newFrame !== frameNumber) {
+            frameNumber = newFrame;
+            loadImage();
+        }
+    }
+
+    function handlePan(event) {
+        const { deltaX } = event;
+        swipePosition = Math.min(Math.max(0, swipePosition + (deltaX / 20)), 10)
+        updateFrame();
+    }
+
+    // Set up and tear down the scroll event listener
+    onMount(() => {
+      window.onwheel = (event) => {
+          handlePan(event);
+        event.stopPropagation();
+      };
+        loadImage();
+    });
+</script>
+
+<div class="frame-container">
+    {#if imageSrc}
+        <img draggable={false} src={imageSrc} alt="Video Frame" />
+    {:else}
+        <p>Loading frame...</p>
+    {/if}
+</div>
+
+<style>
+    .frame-container {
+        /* Your styling here */
+        overflow: hidden;
+        user-select: none;
+    }
+    img {
+        /* Style for the image */
+    }
+</style>
+

src/styles.css

@@ -14,9 +14,17 @@
   -webkit-text-size-adjust: 100%;
 }
 
+body {
+  margin: 0;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  overflow: hidden;
+  overscroll-behavior-x: contain;
+}
+
 .container {
   margin: 0;
-  padding-top: 10vh;
   display: flex;
   flex-direction: column;
   justify-content: center;
@@ -88,6 +96,13 @@ button {
   margin-right: 5px;
 }
 
+img {
+  width: 100%;
+  height: 100vh;
+  object-fit: contain;
+  user-select: none;
+}
+
 @media (prefers-color-scheme: dark) {
   :root {
     color: #f6f6f6;