From 211d1cc0c1e9e3df54bf295ee258ff27cbdf8929 Mon Sep 17 00:00:00 2001 From: "Andrew C. Freeman" Date: Mon, 8 Jan 2024 15:33:03 -0500 Subject: [PATCH] Add support for Prophesee event cameras (#105) * prophesee utils * prophesee data read * till parse_data * working prophesee reader * prophesee utils * prophesee data read * till parse_data * working prophesee reader * Scaffolding for prophesee transocder * Most of the consume() function * Incorporate prophesee transcoder in adder-viz * Initialize the transcode with a gray-level event for all pixels * Some fixes for playback * Fix header height/width parsing * Fix ln memory scaling * Start fixing intensity reconstruction * Bigger default compression interval * Cleanup * Fix player memory leak * Fix write invoke * Safety mechanisms * Framer fix with buffer limit * Fixes for tests and file encoding to allow for different settings of adu_interval * Player speed fixes * Player buffer limit fixes * Cleanup * Add feature detection for prophesee * Fix feature display * Davis write_out fix * Remove standalone data reader --------- Co-authored-by: arghasen10 --- .gitignore | 5 +- .idea/adder-codec-rs.iml | 1 + .../Run_adder_viz_dynamic.xml | 4 +- adder-codec-core/src/codec/compressed/mod.rs | 1 + .../compressed/source_model/cabac_contexts.rs | 6 +- .../event_structure/event_cube.rs | 11 +- .../src/codec/compressed/stream.rs | 47 +- adder-codec-core/src/codec/decoder.rs | 34 +- adder-codec-core/src/codec/encoder.rs | 22 +- adder-codec-core/src/codec/header.rs | 7 + adder-codec-core/src/codec/mod.rs | 9 +- adder-codec-core/src/lib.rs | 2 +- adder-codec-core/tests/integration_tests.rs | 5 + .../examples/framed_video_to_adder.rs | 1 + adder-codec-rs/src/bin/adder_simulproc.rs | 2 + adder-codec-rs/src/framer/driver.rs | 15 +- adder-codec-rs/src/raw/stream.rs | 2 +- adder-codec-rs/src/transcoder/source/davis.rs | 13 +- .../src/transcoder/source/framed.rs | 2 + adder-codec-rs/src/transcoder/source/mod.rs | 1 + .../src/transcoder/source/prophesee.rs | 545 ++++++++++++++++++ adder-codec-rs/src/transcoder/source/video.rs | 10 + adder-codec-rs/src/utils/cv.rs | 10 + adder-codec-rs/src/utils/stream_migration.rs | 2 + adder-codec-rs/tests/integration_tests.rs | 3 + adder-viz/src/player/adder.rs | 10 +- adder-viz/src/player/ui.rs | 9 +- adder-viz/src/transcoder/adder.rs | 56 +- adder-viz/src/transcoder/ui.rs | 131 +++-- 29 files changed, 857 insertions(+), 109 deletions(-) create mode 100644 adder-codec-rs/src/transcoder/source/prophesee.rs diff --git a/.gitignore b/.gitignore index 7c3a5867..d0f56ca6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ # Generated by Cargo # will have compiled files and executables -/target/ - +**/target/ +**/*.dat +**/*.py # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock diff --git a/.idea/adder-codec-rs.iml b/.idea/adder-codec-rs.iml index ec0fe771..eab95c41 100644 --- a/.idea/adder-codec-rs.iml +++ b/.idea/adder-codec-rs.iml @@ -29,6 +29,7 @@ + diff --git a/.idea/runConfigurations/Run_adder_viz_dynamic.xml b/.idea/runConfigurations/Run_adder_viz_dynamic.xml index 3abe001b..2677c1b3 100644 --- a/.idea/runConfigurations/Run_adder_viz_dynamic.xml +++ b/.idea/runConfigurations/Run_adder_viz_dynamic.xml @@ -1,6 +1,6 @@ - - \ No newline at end of file + diff --git a/adder-codec-core/src/codec/compressed/mod.rs b/adder-codec-core/src/codec/compressed/mod.rs index b014d919..f27dd763 100644 --- a/adder-codec-core/src/codec/compressed/mod.rs +++ b/adder-codec-core/src/codec/compressed/mod.rs @@ -65,6 +65,7 @@ mod tests { }), ); let meta = *encoder.meta(); + dbg!(meta); encoder.ingest_event(test_event).unwrap(); test_event.t += 100; encoder.ingest_event(test_event).unwrap(); diff --git a/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs b/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs index 84c702db..6408da8a 100644 --- a/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs +++ b/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs @@ -92,7 +92,11 @@ impl Contexts { if t_residual_i64.abs() < self.t_residual_max { (0, t_residual_i64) } else { - let actual_dt = event.t - prev_event.t; + if event.t < prev_event.t { + // dbg!(event.clone(), prev_event.clone()); + } + + let actual_dt = event.t.saturating_sub(prev_event.t); let actual_intensity = self.event_to_intensity(event.d, actual_dt, dt_ref); let mut recon_intensity = actual_intensity; let mut bitshift: u8 = 0; diff --git a/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs b/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs index 083b6dd4..a57dd0c0 100644 --- a/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs +++ b/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs @@ -123,8 +123,6 @@ impl HandleEvent for EventCube { event.coord.x -= self.start_x; let item = EventCoordless::from(event); - self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()] - .push(item); if self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()] .len() @@ -134,10 +132,17 @@ impl HandleEvent for EventCube { [event.coord.x_usize()][self.raw_event_lists[event.coord.c_usize()] [event.coord.y_usize()][event.coord.x_usize()] .len() - - 2]; + - 1]; + if event.t <= last.t { + // dbg!(event.t, last.t); + return false; + } debug_assert!(event.t >= last.t); } + self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()] + .push(item); + self.raw_event_memory[event.coord.c_usize()][event.coord.y_usize()] [event.coord.x_usize()] = EventCoordless::from(event); diff --git a/adder-codec-core/src/codec/compressed/stream.rs b/adder-codec-core/src/codec/compressed/stream.rs index c97bcde9..398ecb4b 100644 --- a/adder-codec-core/src/codec/compressed/stream.rs +++ b/adder-codec-core/src/codec/compressed/stream.rs @@ -1,6 +1,6 @@ use crate::codec::{CodecError, CodecMetadata, EncoderOptions, ReadCompression, WriteCompression}; use bitstream_io::{BigEndian, BitRead, BitReader, BitWrite, BitWriter}; -use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::io::{BufWriter, Cursor, Read, Seek, SeekFrom, Write}; use crate::codec::compressed::source_model::event_structure::event_adu::EventAdu; use crate::codec::compressed::source_model::HandleEvent; @@ -27,12 +27,7 @@ pub struct CompressedInput { impl CompressedOutput { /// Create a new compressed output stream. pub fn new(meta: CodecMetadata, writer: W) -> Self { - let adu = EventAdu::new( - meta.plane, - 0, - meta.ref_interval, - (meta.delta_t_max / meta.ref_interval) as usize, - ); + let adu = EventAdu::new(meta.plane, 0, meta.ref_interval, meta.adu_interval as usize); Self { meta, @@ -132,7 +127,7 @@ impl WriteCompression for CompressedOutput { impl CompressedInput { /// Create a new compressed input stream. - pub fn new(delta_t_max: DeltaT, ref_interval: DeltaT) -> Self + pub fn new(delta_t_max: DeltaT, ref_interval: DeltaT, adu_interval: usize) -> Self where Self: Sized, { @@ -147,6 +142,7 @@ impl CompressedInput { delta_t_max, event_size: 0, source_camera: Default::default(), + adu_interval, }, adu: None, _phantom: std::marker::PhantomData, @@ -186,7 +182,7 @@ impl ReadCompression for CompressedInput { self.meta.plane, 0, self.meta.ref_interval, - (self.meta.delta_t_max / self.meta.ref_interval) as usize, + self.meta.adu_interval, )); } @@ -278,9 +274,10 @@ mod tests { }, tps: 7650, ref_interval: dt_ref, - delta_t_max: dt_ref * num_intervals as u32, + delta_t_max: dt_ref * num_intervals, event_size: 0, source_camera: SourceCamera::FramedU8, + adu_interval: num_intervals as usize, }, Cursor::new(Vec::new()), ); @@ -338,9 +335,10 @@ mod tests { }, tps: 7650, ref_interval: dt_ref, - delta_t_max: dt_ref * num_intervals as u32, + delta_t_max: dt_ref * num_intervals, event_size: 0, source_camera: SourceCamera::FramedU8, + adu_interval: num_intervals as usize, }, Cursor::new(Vec::new()), ); @@ -385,7 +383,11 @@ mod tests { // Check that the size is less than the raw events assert!((output.len() as u32) < counter * 9); - let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref); + let mut compressed_input = CompressedInput::new( + dt_ref * num_intervals as u32, + dt_ref, + num_intervals as usize, + ); compressed_input.meta.plane = plane; let mut stream = BitReader::endian(Cursor::new(output), BigEndian); for _ in 0..counter - 1 { @@ -431,6 +433,7 @@ mod tests { delta_t_max: dt_ref * num_intervals as u32, event_size: 0, source_camera: SourceCamera::FramedU8, + adu_interval: num_intervals as usize, }, Cursor::new(Vec::new()), ); @@ -459,7 +462,11 @@ mod tests { // Check that the size is less than the raw events assert!((output.len() as u32) < counter * 9); - let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref); + let mut compressed_input = CompressedInput::new( + dt_ref * num_intervals as u32, + dt_ref, + num_intervals as usize, + ); compressed_input.meta.plane = plane; let mut stream = BitReader::endian(Cursor::new(output), BigEndian); for _ in 0..counter - 1 { @@ -514,6 +521,7 @@ mod tests { delta_t_max: dt_ref * num_intervals as u32, event_size: 0, source_camera: SourceCamera::FramedU8, + adu_interval: num_intervals as usize, }, Cursor::new(Vec::new()), ); @@ -571,7 +579,11 @@ mod tests { assert!(!output.is_empty()); // Check that the size is less than the raw events - let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref); + let mut compressed_input = CompressedInput::new( + dt_ref * num_intervals as u32, + dt_ref, + num_intervals as usize, + ); compressed_input.meta.plane = plane; let mut stream = BitReader::endian(Cursor::new(output), BigEndian); for _ in 0..counter + 1 { @@ -624,6 +636,7 @@ mod tests { delta_t_max: dt_ref * num_intervals as u32, event_size: 0, source_camera: SourceCamera::FramedU8, + adu_interval: num_intervals as usize, }, Cursor::new(Vec::new()), ); @@ -688,7 +701,11 @@ mod tests { // Check that the size is less than the raw events assert!((output.len() as u32) < counter * 9); - let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref); + let mut compressed_input = CompressedInput::new( + dt_ref * num_intervals as u32, + dt_ref, + num_intervals as usize, + ); compressed_input.meta.plane = plane; let mut stream = BitReader::endian(Cursor::new(output), BigEndian); loop { diff --git a/adder-codec-core/src/codec/decoder.rs b/adder-codec-core/src/codec/decoder.rs index e7c2cd6a..43bf850d 100644 --- a/adder-codec-core/src/codec/decoder.rs +++ b/adder-codec-core/src/codec/decoder.rs @@ -10,7 +10,8 @@ use crate::codec::compressed::stream::CompressedInput; use crate::codec::encoder::Encoder; use crate::codec::header::{ - EventStreamHeader, EventStreamHeaderExtensionV1, EventStreamHeaderExtensionV2, MAGIC_COMPRESSED, + EventStreamHeader, EventStreamHeaderExtensionV1, EventStreamHeaderExtensionV2, + EventStreamHeaderExtensionV3, MAGIC_COMPRESSED, }; use crate::codec::raw::stream::RawInput; use crate::codec::CodecError::Deserialize; @@ -93,7 +94,7 @@ impl Decoder { SourceCamera::FramedU64 => U64, SourceCamera::FramedF32 => F32, SourceCamera::FramedF64 => F64, - SourceCamera::Dvs => F64, + SourceCamera::Dvs => U8, SourceCamera::DavisU8 => U8, SourceCamera::Atis => U8, SourceCamera::Asint => F64, @@ -128,7 +129,8 @@ impl Decoder { ref_interval: header.ref_interval, delta_t_max: header.delta_t_max, event_size: header.event_size, - source_camera: Default::default(), + source_camera: Default::default(), // Gets filled by decoding the V2 header extension + adu_interval: Default::default(), // Gets filled by decoding the V3 header extension }; // Manual fix for malformed files with old software @@ -183,6 +185,23 @@ impl Decoder { return Ok(()); } + extension_size = bincode::serialized_size(&EventStreamHeaderExtensionV3::default())?; + buffer = vec![0; extension_size as usize]; + reader.read_bytes(&mut buffer)?; + let extension_v3 = match self + .bincode + .deserialize_from::<_, EventStreamHeaderExtensionV3>(&*buffer) + { + Ok(header) => header, + Err(_) => return Err(Deserialize), + }; + self.input.meta_mut().adu_interval = extension_v3.adu_interval as usize; + self.input.meta_mut().header_size += extension_size as usize; + + if codec_version == 3 { + return Ok(()); + } + Err(CodecError::UnsupportedVersion(codec_version)) } @@ -289,6 +308,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -325,6 +345,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -371,6 +392,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -436,7 +458,7 @@ mod tests { let output = setup_encoded_compressed(0); let tmp = Cursor::new(&*output); let bufreader = BufReader::new(tmp); - let compression = CompressedInput::new(255, 255); + let compression = CompressedInput::new(255, 255, 1); let mut bitreader = BitReader::endian(bufreader, BigEndian); let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap(); @@ -449,7 +471,7 @@ mod tests { let output = setup_encoded_compressed(1); let tmp = Cursor::new(&*output); let bufreader = BufReader::new(tmp); - let compression = CompressedInput::new(255, 255); + let compression = CompressedInput::new(255, 255, 1); let mut bitreader = BitReader::endian(bufreader, BigEndian); let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap(); @@ -462,7 +484,7 @@ mod tests { let output = setup_encoded_compressed(2); let tmp = Cursor::new(&*output); let bufreader = BufReader::new(tmp); - let compression = CompressedInput::new(255, 255); + let compression = CompressedInput::new(255, 255, 1); let mut bitreader = BitReader::endian(bufreader, BigEndian); let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap(); diff --git a/adder-codec-core/src/codec/encoder.rs b/adder-codec-core/src/codec/encoder.rs index 03b5092e..69f5515c 100644 --- a/adder-codec-core/src/codec/encoder.rs +++ b/adder-codec-core/src/codec/encoder.rs @@ -18,7 +18,7 @@ use crate::codec::compressed::stream::CompressedOutput; use crate::codec::empty::stream::EmptyOutput; use crate::codec::header::{ EventStreamHeader, EventStreamHeaderExtensionV0, EventStreamHeaderExtensionV1, - EventStreamHeaderExtensionV2, + EventStreamHeaderExtensionV2, EventStreamHeaderExtensionV3, }; use crate::codec::raw::stream::RawOutput; @@ -123,7 +123,7 @@ impl Encoder { SourceCamera::FramedU64 => U64, SourceCamera::FramedF32 => F32, SourceCamera::FramedF64 => F64, - SourceCamera::Dvs => F64, + SourceCamera::Dvs => U8, SourceCamera::DavisU8 => U8, SourceCamera::Atis => U8, SourceCamera::Asint => F64, @@ -216,6 +216,16 @@ impl Encoder { if meta.codec_version == 2 { return Ok(buffer); } + + self.bincode.serialize_into( + &mut buffer, + &EventStreamHeaderExtensionV3 { + adu_interval: meta.adu_interval as u32, + }, + )?; + if meta.codec_version == 3 { + return Ok(buffer); + } Err(CodecError::BadFile) } @@ -328,6 +338,7 @@ mod tests { delta_t_max: 0, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bincode: DefaultOptions::new() .with_fixint_encoding() @@ -366,6 +377,7 @@ mod tests { delta_t_max: 0, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -405,6 +417,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -431,7 +444,7 @@ mod tests { let mut writer = encoder.close_writer().unwrap().unwrap(); writer.flush().unwrap(); let output = writer.into_inner().unwrap(); - assert_eq!(output.len(), 33 + 22); // 33 bytes for the header, 22 bytes for the 2 events + assert_eq!(output.len(), 37 + 22); // 37 bytes for the header, 22 bytes for the 2 events } #[test] @@ -450,6 +463,7 @@ mod tests { delta_t_max: 0, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, // frame: Default::default(), // adu: Adu::new(), @@ -484,6 +498,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: Default::default(), }, bufwriter, ); @@ -513,6 +528,7 @@ mod tests { delta_t_max: 255, event_size: 0, source_camera: Default::default(), + adu_interval: Default::default(), }, bufwriter, ); diff --git a/adder-codec-core/src/codec/header.rs b/adder-codec-core/src/codec/header.rs index 2ee11393..c08a6608 100644 --- a/adder-codec-core/src/codec/header.rs +++ b/adder-codec-core/src/codec/header.rs @@ -39,7 +39,14 @@ impl HeaderExtension for EventStreamHeaderExtensionV1 {} pub(crate) struct EventStreamHeaderExtensionV2 { pub(crate) time_mode: TimeMode, } + +#[derive(Debug, Default, Serialize, Deserialize)] +pub(crate) struct EventStreamHeaderExtensionV3 { + pub(crate) adu_interval: u32, +} + impl HeaderExtension for EventStreamHeaderExtensionV2 {} +impl HeaderExtension for EventStreamHeaderExtensionV3 {} impl EventStreamHeader { pub(crate) fn new( diff --git a/adder-codec-core/src/codec/mod.rs b/adder-codec-core/src/codec/mod.rs index 8973c4c3..a24d2001 100644 --- a/adder-codec-core/src/codec/mod.rs +++ b/adder-codec-core/src/codec/mod.rs @@ -70,11 +70,11 @@ pub mod raw; /// Current latest version of the codec. /// /// This is the version which will be written to the header. -pub const LATEST_CODEC_VERSION: u8 = 2; +pub const LATEST_CODEC_VERSION: u8 = 3; /// The metadata which stays the same over the course of an ADΔER stream #[allow(missing_docs)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] pub struct CodecMetadata { pub codec_version: u8, pub header_size: usize, @@ -85,6 +85,7 @@ pub struct CodecMetadata { pub delta_t_max: DeltaT, pub event_size: u8, pub source_camera: SourceCamera, + pub adu_interval: usize, // TODO: Allow the adu_interval to be non-constant. Each ADU will encode its own size at its beginning } impl Default for CodecMetadata { @@ -99,6 +100,7 @@ impl Default for CodecMetadata { delta_t_max: 255, event_size: 9, source_camera: Default::default(), + adu_interval: 1, } } } @@ -256,7 +258,8 @@ pub enum CodecError { Encoder options below */ -/// Options related to encoder controls +/// Options related to encoder controls (what gets encoded and how) +/// TODO: Move adu_interval into this, rather than be fixed for the whole compressed file #[derive(Copy, Clone, PartialEq, Debug)] pub struct EncoderOptions { /// Allow the encoder to randomly drop events before compressing, if the event rate is too high diff --git a/adder-codec-core/src/lib.rs b/adder-codec-core/src/lib.rs index 13f0eb67..3bbd11dc 100644 --- a/adder-codec-core/src/lib.rs +++ b/adder-codec-core/src/lib.rs @@ -477,7 +477,7 @@ pub fn open_file_decoder( #[cfg(feature = "compression")] { bufreader = BufReader::new(File::open(file_path)?); - let compression = CompressedInput::new(0, 0); // TODO: temporary args. Need to refactor. + let compression = CompressedInput::new(0, 0, 0); // TODO: temporary args. Need to refactor. bitreader = BitReader::endian(bufreader, BigEndian); Decoder::new_compressed(compression, &mut bitreader)? } diff --git a/adder-codec-core/tests/integration_tests.rs b/adder-codec-core/tests/integration_tests.rs index 64e4cd93..ac4c978a 100644 --- a/adder-codec-core/tests/integration_tests.rs +++ b/adder-codec-core/tests/integration_tests.rs @@ -22,10 +22,13 @@ fn test_read_adder_raw() -> Result<(), Box> { fn test_build_first_frame() -> Result<(), Box> { // Open the virat_small_gray.adder sample file as a RawInput let (mut stream, mut bitreader) = open_file_decoder("tests/samples/virat_small_gray.adder")?; + stream.meta_mut().adu_interval = + (stream.meta().delta_t_max / stream.meta().ref_interval) as usize; // This is a fix since we're reading a v2-encoded file // Create the compressed encoder let bufwriter = BufWriter::new(vec![]); let compression = CompressedOutput::new(*stream.meta(), bufwriter); + let mut encoder: Encoder>> = Encoder::new_compressed(compression, EncoderOptions::default((stream.meta()).plane)); @@ -42,6 +45,8 @@ fn test_build_first_frame() -> Result<(), Box> { fn test_build_many_frames() -> Result<(), Box> { // Open the virat_small_gray.adder sample file as a RawInput let (mut stream, mut bitreader) = open_file_decoder("tests/samples/virat_small_gray.adder")?; + stream.meta_mut().adu_interval = + (stream.meta().delta_t_max / stream.meta().ref_interval) as usize; // This is a fix since we're reading a v2-encoded file // Create the compressed encoder let bufwriter = BufWriter::new(vec![]); diff --git a/adder-codec-rs/examples/framed_video_to_adder.rs b/adder-codec-rs/examples/framed_video_to_adder.rs index 3dca470c..8bad21c4 100644 --- a/adder-codec-rs/examples/framed_video_to_adder.rs +++ b/adder-codec-rs/examples/framed_video_to_adder.rs @@ -28,6 +28,7 @@ fn main() -> Result<(), Box> { FramedU8, TimeMode::DeltaT, PixelMultiMode::Normal, + Some(30), EncoderType::Raw, EncoderOptions::default(plane), writer, diff --git a/adder-codec-rs/src/bin/adder_simulproc.rs b/adder-codec-rs/src/bin/adder_simulproc.rs index f4e04d3c..569efed6 100644 --- a/adder-codec-rs/src/bin/adder_simulproc.rs +++ b/adder-codec-rs/src/bin/adder_simulproc.rs @@ -84,6 +84,7 @@ async fn main() -> Result<(), Box> { FramedU8, time_mode, integration_mode, + Some((args.delta_t_max / args.ref_time) as usize), EncoderType::Compressed, EncoderOptions::default(plane), BufWriter::new(file), @@ -211,6 +212,7 @@ mod tests { FramedU8, TimeMode::DeltaT, PixelMultiMode::Normal, + None, EncoderType::Raw, EncoderOptions::default(plane), writer, diff --git a/adder-codec-rs/src/framer/driver.rs b/adder-codec-rs/src/framer/driver.rs index 8597a812..96d4306d 100644 --- a/adder-codec-rs/src/framer/driver.rs +++ b/adder-codec-rs/src/framer/driver.rs @@ -804,6 +804,14 @@ impl + Serialize> FrameSequence { /// Get whether or not the next frame is "filled" (i.e., all pixels have been written to) #[must_use] pub fn is_frame_0_filled(&self) -> bool { + if let Some(buffer_limit) = self.buffer_limit { + for chunk in self.frames.iter() { + if chunk.len() > buffer_limit as usize { + return true; + } + } + } + for chunk in self.chunk_filled_tracker.iter() { if !chunk { return false; @@ -1084,13 +1092,18 @@ fn ingest_event_for_chunk< } if let Some(buffer_limit) = buffer_limit { + // dbg!("buffer filled"); if *last_filled_frame_ref > state.frames_written + buffer_limit as i64 { + // dbg!("buffer filled 2"); frame_chunk[0].filled_count = frame_chunk[0].array.len(); } } debug_assert!(*last_filled_frame_ref >= 0); - debug_assert!(frame_chunk[0].filled_count <= frame_chunk[0].array.len()); + if frame_chunk[0].filled_count > frame_chunk[0].array.len() { + frame_chunk[0].filled_count = frame_chunk[0].array.len(); + } + ( frame_chunk[0].filled_count == frame_chunk[0].array.len(), grew, diff --git a/adder-codec-rs/src/raw/stream.rs b/adder-codec-rs/src/raw/stream.rs index a43dad8b..a0db01de 100644 --- a/adder-codec-rs/src/raw/stream.rs +++ b/adder-codec-rs/src/raw/stream.rs @@ -104,7 +104,7 @@ impl Codec for Raw { SourceCamera::FramedU64 => U64, SourceCamera::FramedF32 => F32, SourceCamera::FramedF64 => F64, - SourceCamera::Dvs => F64, + SourceCamera::Dvs => U8, SourceCamera::DavisU8 => U8, SourceCamera::Atis => U8, SourceCamera::Asint => F64, diff --git a/adder-codec-rs/src/transcoder/source/davis.rs b/adder-codec-rs/src/transcoder/source/davis.rs index 44653599..645b5d0a 100644 --- a/adder-codec-rs/src/transcoder/source/davis.rs +++ b/adder-codec-rs/src/transcoder/source/davis.rs @@ -30,6 +30,7 @@ use adder_codec_core::{Event, PlaneSize, SourceCamera, SourceType, TimeMode}; use crate::framer::scale_intensity::{FrameValue, SaeTime}; use crate::transcoder::event_pixel_tree::Intensity32; +use crate::utils::cv::clamp_u8; use crate::utils::viz::ShowFeatureMode; use tokio::runtime::Runtime; use video_rs_adder_dep::Frame; @@ -1021,6 +1022,7 @@ impl VideoBuilder for Davis { source_camera: SourceCamera, time_mode: TimeMode, pixel_multi_mode: PixelMultiMode, + adu_interval: Option, encoder_type: EncoderType, encoder_options: EncoderOptions, write: W, @@ -1029,6 +1031,7 @@ impl VideoBuilder for Davis { Some(source_camera), Some(time_mode), Some(pixel_multi_mode), + adu_interval, encoder_type, encoder_options, write, @@ -1060,16 +1063,6 @@ fn check_dvs_after(dvs_event_t: i64, timestamp_after: i64) -> bool { dvs_event_t > timestamp_after } -fn clamp_u8(frame_val: &mut f64, last_val_ln: &mut f64) { - if *frame_val <= 0.0 { - *frame_val = 0.0; - *last_val_ln = 0.0; // = 0.0_f64.ln_1p(); - } else if *frame_val > 255.0 { - *frame_val = 255.0; - *last_val_ln = 255.0_f64.ln_1p(); - } -} - /// Get the next APS image from the video source. /// Returns a tuple of the image, the timestamp of the image, the timestamp of the end of the /// frame, and the events occurring during the interval. diff --git a/adder-codec-rs/src/transcoder/source/framed.rs b/adder-codec-rs/src/transcoder/source/framed.rs index 89e51814..1cba3267 100644 --- a/adder-codec-rs/src/transcoder/source/framed.rs +++ b/adder-codec-rs/src/transcoder/source/framed.rs @@ -258,6 +258,7 @@ impl VideoBuilder for Framed { source_camera: SourceCamera, time_mode: TimeMode, pixel_multi_mode: PixelMultiMode, + adu_interval: Option, encoder_type: EncoderType, encoder_options: EncoderOptions, write: W, @@ -266,6 +267,7 @@ impl VideoBuilder for Framed { Some(source_camera), Some(time_mode), Some(pixel_multi_mode), + adu_interval, encoder_type, encoder_options, write, diff --git a/adder-codec-rs/src/transcoder/source/mod.rs b/adder-codec-rs/src/transcoder/source/mod.rs index 6139c5d9..fc77edd3 100644 --- a/adder-codec-rs/src/transcoder/source/mod.rs +++ b/adder-codec-rs/src/transcoder/source/mod.rs @@ -7,5 +7,6 @@ pub mod framed; /// Common functions and structs for all transcoder sources pub mod video; +pub mod prophesee; diff --git a/adder-codec-rs/src/transcoder/source/prophesee.rs b/adder-codec-rs/src/transcoder/source/prophesee.rs new file mode 100644 index 00000000..dd6c58f3 --- /dev/null +++ b/adder-codec-rs/src/transcoder/source/prophesee.rs @@ -0,0 +1,545 @@ +use crate::framer::scale_intensity::{FrameValue, SaeTime}; +use crate::transcoder::source::video::FramedViewMode::SAE; +use crate::transcoder::source::video::{ + integrate_for_px, Source, SourceError, Video, VideoBuilder, +}; +use crate::utils::cv::clamp_u8; +use crate::utils::viz::ShowFeatureMode; +use adder_codec_core::codec::{EncoderOptions, EncoderType}; +use adder_codec_core::Mode::Continuous; +use adder_codec_core::{ + DeltaT, Event, PixelMultiMode, PlaneSize, SourceCamera, SourceType, TimeMode, +}; +use ndarray::Array3; +use rayon::ThreadPool; +use serde::Deserialize; +use std::error::Error; +use std::fs::File; +use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::Path; +use std::path::PathBuf; +use std::str::FromStr; +use tokio::io::BufWriter; +use video_rs_adder_dep::Frame; + +/// The temporal granularity of the source (ticks per second) +const PROPHESEE_SOURCE_TPS: u32 = 1000000; + +/// Attributes of a framed video -> ADΔER transcode +pub struct Prophesee { + pub(crate) video: Video, + + input_reader: BufReader, + + // We scale up the input timestamps by this amount for accuracy under ADDER. + // For example, with time_change = 255, a timestamp of 12 in the source becomes 3060 + // ADDER ticks + // time_change: u32, + num_dvs_events: usize, + + running_t: u32, + + /// The timestamp (in-camera) of the last DVS event integrated for each pixel + pub dvs_last_timestamps: Array3, + + /// The log-space last intensity value for each pixel + pub dvs_last_ln_val: Array3, + + camera_theta: f64, +} + +#[derive(Debug, Deserialize, Clone)] +struct DvsEvent { + t: u32, + x: u16, + y: u16, + p: u8, +} + +unsafe impl Sync for Prophesee {} + +impl Prophesee { + /// Create a new `Prophesee` transcoder + pub fn new(ref_time: u32, input_filename: String) -> Result> { + let source = File::open(PathBuf::from(input_filename))?; + let mut input_reader = BufReader::new(source); + + // Parse header + let (_, _, _, size) = parse_header(&mut input_reader).unwrap(); + + let plane = PlaneSize::new(size.1 as u16, size.0 as u16, 1)?; + + let mut video = Video::new(plane, Continuous, None)? + .chunk_rows(1) + // Override the tps to assume the source has a temporal granularity of 1000000/second + // The `ref_time` in this case scales up the temporal granularity of the source. + // For example, with ref_time = 20, a timestamp of 12 in the source becomes 240 + // ADDER ticks + .time_parameters( + ref_time * PROPHESEE_SOURCE_TPS, + ref_time, + ref_time * 2, + Some(TimeMode::AbsoluteT), + )?; + + let start_intensities = vec![128_u8; video.state.plane.volume()]; + video.state.running_intensities = Array3::from_shape_vec( + (plane.h().into(), plane.w().into(), plane.c().into()), + start_intensities, + )?; + video.display_frame_features = video.state.running_intensities.clone(); + + let timestamps = vec![2_u32; video.state.plane.volume()]; + + let dvs_last_timestamps: Array3 = Array3::from_shape_vec( + (plane.h().into(), plane.w().into(), plane.c().into()), + timestamps, + )?; + + let plane = &video.state.plane; + + let start_vals = vec![0.5_f64.ln(); video.state.plane.volume()]; + + let dvs_last_ln_val: Array3 = Array3::from_shape_vec( + (plane.h() as usize, plane.w() as usize, plane.c() as usize), + start_vals, + )?; + + let prophesee_source = Prophesee { + video, + input_reader, + num_dvs_events: 0, + running_t: 0, + dvs_last_timestamps, + dvs_last_ln_val, + camera_theta: 0.05, // A fixed assumption + }; + + Ok(prophesee_source) + } +} + +impl Source for Prophesee { + fn consume( + &mut self, + view_interval: u32, + thread_pool: &ThreadPool, + ) -> Result>, SourceError> { + if self.running_t == 0 { + self.video.integrate_matrix( + self.video.state.running_intensities.clone(), + self.video.state.params.ref_time as f32, + 1, + )?; + let first_events: Vec = self + .video + .integrate_matrix( + self.video.state.running_intensities.clone(), + self.video.state.params.ref_time as f32, + 1, + )? + .into_iter() + .flatten() + .collect(); + assert_eq!(first_events.len(), self.video.state.plane.volume()); + self.running_t = 2; + } + + // TODO hardcoded: scale the view interval to be 60 FPS GUI display + let view_interval = (PROPHESEE_SOURCE_TPS / 60); + + // Read events from the source file until we find a timestamp that exceeds our `running_t` + // by at least `view_interval` + let mut dvs_events: Vec = Vec::new(); + let mut dvs_event; + let start_running_t = self.running_t; + loop { + // TODO: integrate to fill in the rest of time once the eof is reached + + dvs_event = match decode_event(&mut self.input_reader) { + Ok(dvs_event) => { + if dvs_event.t > self.running_t { + self.running_t = dvs_event.t; + } + dvs_event + } + Err(e) => { + end_events(self); + return Err(e.into()); + } + }; + dvs_events.push(dvs_event); + if dvs_events.last().unwrap().t > start_running_t + view_interval { + break; + } + } + + let mut events: Vec = Vec::new(); + let crf_parameters = *self.video.encoder.options.crf.get_parameters(); + + // For every dvs event in our queue, integrate the previously seen intensity for all the + // time between the pixel's last input and the current event + for dvs_event in dvs_events { + let x = dvs_event.x as usize; + let y = dvs_event.y as usize; + let p = dvs_event.p as usize; + let t = dvs_event.t; + + // Get the last timestamp for this pixel + let last_t = self.dvs_last_timestamps[[y, x, 0]]; + + if t < last_t { + // dbg!("skipping event"); + continue; + } + + // Get the last ln intensity for this pixel + let mut last_ln_val = self.dvs_last_ln_val[[y, x, 0]]; + + // Convert the ln intensity to a linear intensity + let mut last_val = (last_ln_val.exp() - 1.0) * 255.0; + + clamp_u8(&mut last_val, &mut last_ln_val); + + let px = &mut self.video.event_pixel_trees[[y, x, 0]]; + + if t > last_t + 1 { + // Integrate the last intensity for this pixel over the time since the last event + let time_spanned = ((t - last_t - 1) * self.video.state.params.ref_time); + let intensity_to_integrate = last_val * (t - last_t - 1) as f64; + + let mut base_val = 0; + let _ = integrate_for_px( + px, + &mut base_val, + last_val as u8, + intensity_to_integrate as f32, + time_spanned as f32, + &mut events, + &self.video.state.params, + &crf_parameters, + ); + } + + // Get the new ln intensity + let mut new_ln_val = match p { + 0 => last_ln_val - self.camera_theta, + 1 => last_ln_val + self.camera_theta, + _ => panic!("Invalid polarity"), + }; + + let mut new_val = (new_ln_val.exp() - 1.0) * 255.0; + + clamp_u8(&mut new_val, &mut new_ln_val); + + // Update the last intensity for this pixel + self.dvs_last_ln_val[[y, x, 0]] = new_ln_val; + + // Update the last timestamp for this pixel + self.dvs_last_timestamps[[y, x, 0]] = t; + + if t > last_t { + // Integrate 1 source time unit of the new intensity + let time_spanned = self.video.state.params.ref_time; + let intensity_to_integrate = new_val; + + let mut base_val = 0; + let _ = integrate_for_px( + px, + &mut base_val, + new_val as u8, + intensity_to_integrate as f32, + time_spanned as f32, + &mut events, + &self.video.state.params, + &crf_parameters, + ); + } + + // Update the running intensity for this pixel + if let Some(event) = px.arena[0].best_event { + self.video.state.running_intensities[[y, x, 0]] = u8::get_frame_value( + &event.into(), + SourceType::U8, + self.video.state.params.ref_time as f64, + 32.0, + self.video.state.params.delta_t_max, + self.video.instantaneous_view_mode, + if self.video.instantaneous_view_mode == SAE { + Some(SaeTime { + running_t: px.running_t as DeltaT, + last_fired_t: px.last_fired_t as DeltaT, + }) + } else { + None + }, + ); + self.video.display_frame_features[[y, x, 0]] = + self.video.state.running_intensities[[y, x, 0]]; + }; + } + + if self.video.state.feature_detection { + self.video.display_frame_features = self.video.state.running_intensities.clone(); + } + + // It's expected that the function will spatially parallelize the integrations. With sparse + // data, though, this could be pretty wasteful. For now, just wrap the vec in another vec. + let events_nested: Vec> = vec![events]; + + self.video.handle_features(&events_nested)?; + + for events in &events_nested { + for event in events { + self.video.encoder.ingest_event(*event)?; + } + } + + Ok(events_nested) + } + + fn crf(&mut self, crf: u8) { + self.video.update_crf(crf); + } + + fn get_video_mut(&mut self) -> &mut Video { + &mut self.video + } + + fn get_video_ref(&self) -> &Video { + &self.video + } + + fn get_video(self) -> Video { + self.video + } + + fn get_input(&self) -> Option<&Frame> { + None + } + + fn get_running_input_bitrate(&self) -> f64 { + // TODO + 0.0 + } +} + +fn end_events(prophesee: &mut Prophesee) { + let mut events: Vec = Vec::new(); + let crf_parameters = *prophesee.video.encoder.options.crf.get_parameters(); + + for y in 0..prophesee.video.state.plane.h_usize() { + for x in 0..prophesee.video.state.plane.w_usize() { + let px = &mut prophesee.video.event_pixel_trees[[y, x, 0]]; + let mut base_val = 0; + + // Get the last ln intensity for this pixel + let mut last_ln_val = prophesee.dvs_last_ln_val[[y, x, 0]]; + + // Convert the ln intensity to a linear intensity + let mut last_val = (last_ln_val.exp() - 1.0) * 255.0; + + assert!(prophesee.running_t - prophesee.dvs_last_timestamps[[y, x, 0]] > 0); + + // Integrate the last intensity for this pixel over the time since the last event + let time_spanned = ((prophesee.running_t - prophesee.dvs_last_timestamps[[y, x, 0]]) + * prophesee.video.state.params.ref_time); + let intensity_to_integrate = last_val * time_spanned as f64; + + let _ = integrate_for_px( + px, + &mut base_val, + last_val as u8, + intensity_to_integrate as f32, + time_spanned as f32, + &mut events, + &prophesee.video.state.params, + &crf_parameters, + ); + } + } + + for event in &events { + prophesee.video.encoder.ingest_event(*event).unwrap(); + } +} + +fn parse_header(file: &mut BufReader) -> io::Result<(u64, u8, u8, (u32, u32))> { + file.seek(SeekFrom::Start(0))?; // Seek to the beginning of the file + let mut bod = 0; + let mut end_of_header = false; + let mut num_comment_line = 0; + let mut size = [None, None]; + + // Parse header + while !end_of_header { + bod = file.seek(SeekFrom::Current(0))?; // Get the current position + let mut line = Vec::new(); // Change to Vec + file.read_until(b'\n', &mut line)?; // Read until newline as binary data + if line.is_empty() || line[0] != b'%' { + end_of_header = true; + } else { + let mut words: Vec<&[u8]> = line.split(|&x| x == b' ' || x == b'\t').collect(); // Use &[u8] instead of &str + + if words.len() > 1 { + match words[1] { + b"Height" => { + size[0] = line_to_hw(words); + } + b"Width" => { + size[1] = line_to_hw(words); + } + _ => {} + } + } + num_comment_line += 1; + } + } + + // Parse data + file.seek(SeekFrom::Start(bod))?; // Seek back to the position after the header + let (ev_type, ev_size) = if num_comment_line > 0 { + // Read event type and size + let mut buf = [0; 2]; // Adjust the buffer size based on your data size + file.read_exact(&mut buf)?; + let ev_type = buf[0]; + let ev_size = buf[1]; + + (ev_type, ev_size) + } else { + (0, 0) // Placeholder values, replace with actual logic + }; + bod = file.seek(SeekFrom::Current(0))?; + Ok(( + bod, + ev_type, + ev_size, + (size[0].unwrap_or(70), size[1].unwrap_or(100)), + )) +} + +fn line_to_hw(words: Vec<&[u8]>) -> Option { + let mut word = words.get(2).unwrap(); + let mut new_word = *word; + if *word.last().unwrap() == '\n' as u8 { + // Remove the trailing newline + new_word = &word[..word.len() - 1]; + } + std::str::from_utf8(new_word) + .ok() + .and_then(|s| s.parse().ok()) +} + +fn decode_event(reader: &mut BufReader) -> io::Result<(DvsEvent)> { + // Read one record + let mut buffer = [0; 8]; // Adjust this size to match your record size + reader.read_exact(&mut buffer)?; + + // Interpret the bytes as 't' and 'data' + let t = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); + let data = i32::from_le_bytes([buffer[4], buffer[5], buffer[6], buffer[7]]); + + // Perform bitwise operations + let x = (data & 16383) as u16; + let y = ((data & 268419072) >> 14) as u16; + let p = ((data & 268435456) >> 28) as u8; + + Ok(DvsEvent { t, x, y, p }) +} + +impl VideoBuilder for Prophesee { + fn contrast_thresholds(mut self, c_thresh_pos: u8, _c_thresh_neg: u8) -> Self { + self.video = self.video.c_thresh_pos(c_thresh_pos); + // self.video = self.video.c_thresh_neg(c_thresh_neg); + self + } + + fn crf(mut self, crf: u8) -> Self { + self.video.update_crf(crf); + self + } + + fn quality_manual( + mut self, + c_thresh_baseline: u8, + c_thresh_max: u8, + delta_t_max_multiplier: u32, + c_increase_velocity: u8, + feature_c_radius_denom: f32, + ) -> Self { + self.video.update_quality_manual( + c_thresh_baseline, + c_thresh_max, + delta_t_max_multiplier, + c_increase_velocity, + feature_c_radius_denom, + ); + self + } + + fn c_thresh_pos(mut self, c_thresh_pos: u8) -> Self { + self.video = self.video.c_thresh_pos(c_thresh_pos); + self + } + + fn c_thresh_neg(self, _c_thresh_neg: u8) -> Self { + // self.video = self.video.c_thresh_neg(c_thresh_neg); + self + } + + fn chunk_rows(mut self, chunk_rows: usize) -> Self { + self.video = self.video.chunk_rows(chunk_rows); + self + } + + fn time_parameters( + mut self, + tps: DeltaT, + ref_time: DeltaT, + delta_t_max: DeltaT, + time_mode: Option, + ) -> Result { + eprintln!("setting dtref to {}", ref_time); + self.video = self + .video + .time_parameters(tps, ref_time, delta_t_max, time_mode)?; + Ok(self) + } + + fn write_out( + mut self, + source_camera: SourceCamera, + time_mode: TimeMode, + pixel_multi_mode: PixelMultiMode, + adu_interval: Option, + encoder_type: EncoderType, + encoder_options: EncoderOptions, + write: W, + ) -> Result, SourceError> { + self.video = self.video.write_out( + Some(source_camera), + Some(time_mode), + Some(pixel_multi_mode), + adu_interval, + encoder_type, + encoder_options, + write, + )?; + Ok(Box::new(self)) + } + + fn show_display(mut self, show_display: bool) -> Self { + self.video = self.video.show_display(show_display); + self + } + + fn detect_features(mut self, detect_features: bool, show_features: ShowFeatureMode) -> Self { + self.video = self.video.detect_features(detect_features, show_features); + self + } + + #[cfg(feature = "feature-logging")] + fn log_path(self, _name: String) -> Self { + todo!() + } +} diff --git a/adder-codec-rs/src/transcoder/source/video.rs b/adder-codec-rs/src/transcoder/source/video.rs index 273ed14d..cf0560a9 100644 --- a/adder-codec-rs/src/transcoder/source/video.rs +++ b/adder-codec-rs/src/transcoder/source/video.rs @@ -115,6 +115,10 @@ pub enum SourceError { /// Vision application error #[error("Vision application error")] VisionError(String), + + /// I/O error + #[error("I/O error")] + IoError(#[from] std::io::Error), } #[cfg(feature = "open-cv")] @@ -304,6 +308,7 @@ pub trait VideoBuilder { source_camera: SourceCamera, time_mode: TimeMode, pixel_multi_mode: PixelMultiMode, + adu_interval: Option, encoder_type: EncoderType, encoder_options: EncoderOptions, write: W, @@ -398,6 +403,7 @@ impl Video { delta_t_max: state.params.delta_t_max, event_size: 0, source_camera: SourceCamera::default(), // TODO: Allow for setting this + adu_interval: Default::default(), }; match writer { @@ -546,6 +552,7 @@ impl Video { source_camera: Option, time_mode: Option, pixel_multi_mode: Option, + adu_interval: Option, encoder_type: EncoderType, encoder_options: EncoderOptions, write: W, @@ -567,6 +574,7 @@ impl Video { delta_t_max: self.state.params.delta_t_max, event_size: 0, source_camera: source_camera.unwrap_or_default(), + adu_interval: adu_interval.unwrap_or_default(), }, write, ); @@ -594,6 +602,7 @@ impl Video { delta_t_max: self.state.params.delta_t_max, event_size: 0, source_camera: source_camera.unwrap_or_default(), + adu_interval: Default::default(), }, write, ); @@ -613,6 +622,7 @@ impl Video { delta_t_max: self.state.params.delta_t_max, event_size: 0, source_camera: source_camera.unwrap_or_default(), + adu_interval: Default::default(), }, sink(), ); diff --git a/adder-codec-rs/src/utils/cv.rs b/adder-codec-rs/src/utils/cv.rs index 4afdde54..bd685d5a 100644 --- a/adder-codec-rs/src/utils/cv.rs +++ b/adder-codec-rs/src/utils/cv.rs @@ -417,3 +417,13 @@ fn mean(window: &ArrayView) -> f64 { sum / window.len() as f64 } + +pub fn clamp_u8(frame_val: &mut f64, last_val_ln: &mut f64) { + if *frame_val <= 0.0 { + *frame_val = 0.0; + *last_val_ln = 0.0; // = 0.0_f64.ln_1p(); + } else if *frame_val > 255.0 { + *frame_val = 255.0; + *last_val_ln = 1.0_f64.ln_1p(); + } +} diff --git a/adder-codec-rs/src/utils/stream_migration.rs b/adder-codec-rs/src/utils/stream_migration.rs index b0af6337..38ba5827 100644 --- a/adder-codec-rs/src/utils/stream_migration.rs +++ b/adder-codec-rs/src/utils/stream_migration.rs @@ -125,6 +125,7 @@ mod tests { delta_t_max: 2550, event_size: 0, source_camera: FramedU8, + adu_interval: 1, }, bufwriter, ); @@ -175,6 +176,7 @@ mod tests { delta_t_max: 2550, event_size: 0, source_camera: FramedU8, + adu_interval: 1, }, bufwriter, ); diff --git a/adder-codec-rs/tests/integration_tests.rs b/adder-codec-rs/tests/integration_tests.rs index 6afd3780..bc8debd0 100644 --- a/adder-codec-rs/tests/integration_tests.rs +++ b/adder-codec-rs/tests/integration_tests.rs @@ -261,6 +261,7 @@ fn setup_raw_writer_v0(rand_num: u32) -> Encoder> { delta_t_max: 50000, event_size: 0, source_camera: Default::default(), + adu_interval: 1, }, bufwriter, ); @@ -286,6 +287,7 @@ fn setup_raw_writer_v1(rand_num: u32) -> Encoder> { delta_t_max: 50000, event_size: 0, source_camera: FramedU8, + adu_interval: 1, }, bufwriter, ); @@ -311,6 +313,7 @@ fn setup_raw_writer_v2(rand_num: u32) -> Encoder> { delta_t_max: 50000, event_size: 0, source_camera: FramedU8, + adu_interval: 1, }, bufwriter, ); diff --git a/adder-viz/src/player/adder.rs b/adder-viz/src/player/adder.rs index 4af220ab..d2e99b4c 100644 --- a/adder-viz/src/player/adder.rs +++ b/adder-viz/src/player/adder.rs @@ -90,10 +90,13 @@ impl AdderPlayer { let meta = *stream.meta(); let mut reconstructed_frame_rate = meta.tps as f32 / meta.ref_interval as f32; + if !is_framed(meta.source_camera) { + reconstructed_frame_rate = 60.0; + } reconstructed_frame_rate /= playback_speed; - let framer_builder: FramerBuilder = FramerBuilder::new(meta.plane, 260) + let framer_builder: FramerBuilder = FramerBuilder::new(meta.plane, 1) .codec_version(meta.codec_version, meta.time_mode) .time_parameters( meta.tps, @@ -236,7 +239,10 @@ impl AdderPlayer { let meta = *stream.decoder.meta(); - let frame_length = meta.ref_interval as f64 * self.playback_speed as f64; //TODO: temp + let mut frame_length = meta.ref_interval as f64 * self.playback_speed as f64; //TODO: temp + if !is_framed(meta.source_camera) { + frame_length = meta.tps as f64 / 60.0 * self.playback_speed as f64; + } // if self.view_mode == FramedViewMode::DeltaT { // opencv::core::normalize( diff --git a/adder-viz/src/player/ui.rs b/adder-viz/src/player/ui.rs index 416e7a2a..5dd5967a 100644 --- a/adder-viz/src/player/ui.rs +++ b/adder-viz/src/player/ui.rs @@ -76,7 +76,7 @@ impl Default for PlayerUiState { ui_sliders: Default::default(), ui_sliders_drag: Default::default(), detect_features: false, - buffer_limit: None, + buffer_limit: Some(60), } } } @@ -147,6 +147,7 @@ impl PlayerState { self.ui_info_state.stream_state = stream_state; if let Some(image) = image_opt { + images.remove(&handles.image_view); let handle = images.add(image); handles.image_view = handle; } else if self.ui_info_state.stream_state.file_pos == 1 { @@ -309,7 +310,7 @@ impl PlayerState { let mut buffer_limit = self.ui_state.buffer_limit.unwrap_or(100); let mut buffer_limit_tmp = buffer_limit; - add_slider_row( + need_to_update |= add_slider_row( limit_frame_buffer_bool, false, "Buffer limit:", @@ -326,6 +327,7 @@ impl PlayerState { || self.ui_state.buffer_limit != Some(buffer_limit_tmp)) { self.ui_state.buffer_limit = Some(buffer_limit_tmp); + need_to_update = true; } ui.label("Processing:"); @@ -451,6 +453,9 @@ impl PlayerState { self.ui_state.current_frame = 1; + if let Some(current_rx) = &self.player_rx { + drop(current_rx); + } let (player_tx, player_rx) = bounded(60); let detect_features = self.ui_state.detect_features; diff --git a/adder-viz/src/transcoder/adder.rs b/adder-viz/src/transcoder/adder.rs index 2f419a9a..3720525e 100644 --- a/adder-viz/src/transcoder/adder.rs +++ b/adder-viz/src/transcoder/adder.rs @@ -19,8 +19,9 @@ use adder_codec_rs::davis_edi_rs::util::reconstructor::Reconstructor; use crate::transcoder::ui::{ParamsUiState, TranscoderState}; use adder_codec_rs::adder_codec_core::codec::rate_controller::DEFAULT_CRF_QUALITY; -use adder_codec_rs::adder_codec_core::SourceCamera::{DavisU8, FramedU8}; -use adder_codec_rs::transcoder::source::video::VideoBuilder; +use adder_codec_rs::adder_codec_core::SourceCamera::{DavisU8, Dvs, FramedU8}; +use adder_codec_rs::transcoder::source::prophesee::Prophesee; +use adder_codec_rs::transcoder::source::video::{Source, VideoBuilder}; use bevy_egui::egui::{Color32, RichText}; #[cfg(feature = "open-cv")] use opencv::Result; @@ -30,6 +31,7 @@ pub struct AdderTranscoder { pub(crate) framed_source: Option>>, #[cfg(feature = "open-cv")] pub(crate) davis_source: Option>>, + pub(crate) prophesee_source: Option>>, pub(crate) live_image: Image, } @@ -98,6 +100,7 @@ impl AdderTranscoder { FramedU8, ui_state.time_mode, ui_state.integration_mode_radio_state, + Some(ui_state.delta_t_max_mult as usize), ui_state.encoder_type, ui_state.encoder_options, writer, @@ -110,6 +113,7 @@ impl AdderTranscoder { framed_source: Some(framed), #[cfg(feature = "open-cv")] davis_source: None, + prophesee_source: None, live_image: Default::default(), }) // } @@ -221,6 +225,7 @@ impl AdderTranscoder { DavisU8, ui_state.time_mode, ui_state.integration_mode_radio_state, + Some(ui_state.delta_t_max_mult as usize), ui_state.encoder_type, ui_state.encoder_options, writer, @@ -230,6 +235,53 @@ impl AdderTranscoder { Ok(AdderTranscoder { framed_source: None, davis_source: Some(davis_source), + prophesee_source: None, + live_image: Default::default(), + }) + } + + // Prophesee .dat files + Some(ext) if ext == "dat" => { + let output_string = output_path_opt + .map(|output_path| output_path.to_str().expect("Bad path").to_string()); + + let mut prophesee_source: Prophesee> = Prophesee::new( + ui_state.delta_t_ref as u32, + input_path_buf.to_str().unwrap().to_string(), + )? + .crf( + ui_state + .encoder_options + .crf + .get_quality() + .unwrap_or(DEFAULT_CRF_QUALITY), + ); + let adu_interval = (prophesee_source.get_video_ref().state.tps as f32 + / ui_state.delta_t_ref) + as usize; + + if let Some(output_string) = output_string { + let writer = BufWriter::new(File::create(output_string)?); + prophesee_source = *prophesee_source.write_out( + Dvs, + ui_state.time_mode, + ui_state.integration_mode_radio_state, + Some(adu_interval), + ui_state.encoder_type, + ui_state.encoder_options, + writer, + )?; + } + + ui_state.delta_t_max_mult = + prophesee_source.get_video_ref().get_delta_t_max() + / prophesee_source.get_video_ref().state.params.ref_time as u32; + ui_state.delta_t_max_mult_slider = ui_state.delta_t_max_mult; + Ok(AdderTranscoder { + framed_source: None, + #[cfg(feature = "open-cv")] + davis_source: None, + prophesee_source: Some(prophesee_source), live_image: Default::default(), }) } diff --git a/adder-viz/src/transcoder/ui.rs b/adder-viz/src/transcoder/ui.rs index d4153bda..3ec86c91 100644 --- a/adder-viz/src/transcoder/ui.rs +++ b/adder-viz/src/transcoder/ui.rs @@ -34,7 +34,7 @@ pub struct ParamsUiState { pub(crate) delta_t_ref_max: f32, pub(crate) delta_t_max_mult: u32, delta_t_ref_slider: f32, - delta_t_max_mult_slider: u32, + pub(crate) delta_t_max_mult_slider: u32, pub(crate) scale: f64, scale_slider: f64, pub(crate) thread_count: usize, @@ -256,6 +256,7 @@ impl TranscoderState { if let Some(path) = rfd::FileDialog::new() .add_filter("framed video", &["mp4"]) .add_filter("DVS/DAVIS video", &["aedat4"]) + .add_filter("Prophesee video", &["dat"]) .pick_file() { self.ui_info_state.input_path_0 = Some(path.clone()); @@ -270,7 +271,7 @@ impl TranscoderState { } } - ui.label("OR drag and drop your source file here (.mp4, .aedat4)"); + ui.label("OR drag and drop your source file here (.mp4, .aedat4, .dat)"); }); ui.horizontal(|ui| { @@ -433,53 +434,68 @@ impl TranscoderState { let source: &mut dyn Source> = { match &mut self.transcoder.framed_source { None => { - #[cfg(feature = "open-cv")] - match &mut self.transcoder.davis_source { + match &mut self.transcoder.prophesee_source { None => { - return; - } + #[cfg(feature = "open-cv")] + match &mut self.transcoder.davis_source { + None => { + return; + } - Some(source) => { - if source.mode != self.ui_state.davis_mode_radio_state - || source.get_reconstructor().as_ref().unwrap().output_fps - != self.ui_state.davis_output_fps - || ((source.get_video_ref().get_time_mode() - != self.ui_state.time_mode - || source.get_video_ref().encoder_type - != self.ui_state.encoder_type - || source.get_video_ref().get_encoder_options().event_drop - != self.ui_state.encoder_options.event_drop - || source.get_video_ref().get_encoder_options().event_order - != self.ui_state.encoder_options.event_order - || source.get_video_ref().state.params.pixel_multi_mode - != self.ui_state.integration_mode_radio_state) - && self.ui_info_state.output_path.is_some()) - { - if self.ui_state.davis_mode_radio_state == RawDvs { - // self.ui_state.davis_output_fps = 1000000.0; - // self.ui_state.davis_output_fps_slider = 1000000.0; - self.ui_state.optimize_c = false; + Some(source) => { + if source.mode != self.ui_state.davis_mode_radio_state + || source.get_reconstructor().as_ref().unwrap().output_fps + != self.ui_state.davis_output_fps + || ((source.get_video_ref().get_time_mode() + != self.ui_state.time_mode + || source.get_video_ref().encoder_type + != self.ui_state.encoder_type + || source + .get_video_ref() + .get_encoder_options() + .event_drop + != self.ui_state.encoder_options.event_drop + || source + .get_video_ref() + .get_encoder_options() + .event_order + != self.ui_state.encoder_options.event_order + || source + .get_video_ref() + .state + .params + .pixel_multi_mode + != self.ui_state.integration_mode_radio_state) + && self.ui_info_state.output_path.is_some()) + { + if self.ui_state.davis_mode_radio_state == RawDvs { + // self.ui_state.davis_output_fps = 1000000.0; + // self.ui_state.davis_output_fps_slider = 1000000.0; + self.ui_state.optimize_c = false; + } + replace_adder_transcoder( + self, + self.ui_info_state.input_path_0.clone(), + self.ui_info_state.input_path_1.clone(), + self.ui_info_state.output_path.clone(), + 0, + ); + images.clear(); + return; + } + let tmp = source.get_reconstructor_mut().as_mut().unwrap(); + tmp.set_optimize_c( + self.ui_state.optimize_c, + self.ui_state.optimize_c_frequency, + ); + source } - replace_adder_transcoder( - self, - self.ui_info_state.input_path_0.clone(), - self.ui_info_state.input_path_1.clone(), - self.ui_info_state.output_path.clone(), - 0, - ); - images.clear(); - return; } - let tmp = source.get_reconstructor_mut().as_mut().unwrap(); - tmp.set_optimize_c( - self.ui_state.optimize_c, - self.ui_state.optimize_c_frequency, - ); - source + #[cfg(not(feature = "open-cv"))] + return; } + Some(source) => source, } - #[cfg(not(feature = "open-cv"))] - return; } Some(source) => { if source.scale != self.ui_state.scale @@ -601,22 +617,26 @@ impl TranscoderState { let ui_info_state = &mut self.ui_info_state; ui_info_state.events_per_sec = 0.; + // TODO: The below code is absolutely horrible. let source: &mut dyn Source> = { match &mut self.transcoder.framed_source { - None => { - #[cfg(feature = "open-cv")] - match &mut self.transcoder.davis_source { - None => { - return Ok(()); - } - Some(source) => { - ui_info_state.davis_latency = Some(source.get_latency() as f64); - source + None => match &mut self.transcoder.prophesee_source { + None => { + #[cfg(feature = "open-cv")] + match &mut self.transcoder.davis_source { + None => { + return Ok(()); + } + Some(source) => { + ui_info_state.davis_latency = Some(source.get_latency() as f64); + source + } } + #[cfg(not(feature = "open-cv"))] + return Ok(()); } - #[cfg(not(feature = "open-cv"))] - return Ok(()); - } + Some(source) => source, + }, Some(source) => source, } }; @@ -682,6 +702,7 @@ impl TranscoderState { if let Some(image) = images.get_mut(&handles.image_view) { crate::utils::prep_bevy_image_mut(image_mat, color, image)?; } else { + // dbg!("else"); let image_bevy = prep_bevy_image( image_mat, color,