Add support for Prophesee event cameras (#105)

* prophesee utils * prophesee data read * till parse_data * working prophesee reader * prophesee utils * prophesee data read * till parse_data * working prophesee reader * Scaffolding for prophesee transocder * Most of the consume() function * Incorporate prophesee transcoder in adder-viz * Initialize the transcode with a gray-level event for all pixels * Some fixes for playback * Fix header height/width parsing * Fix ln memory scaling * Start fixing intensity reconstruction * Bigger default compression interval * Cleanup * Fix player memory leak * Fix write invoke * Safety mechanisms * Framer fix with buffer limit * Fixes for tests and file encoding to allow for different settings of adu_interval * Player speed fixes * Player buffer limit fixes * Cleanup * Add feature detection for prophesee * Fix feature display * Davis write_out fix * Remove standalone data reader --------- Co-authored-by: arghasen10 <[email protected]>
ac-freeman · Jan 8, 2024 · 211d1cc · 211d1cc
1 parent 1835d49
commit 211d1cc
Show file tree

Hide file tree

Showing 29 changed files with 857 additions and 109 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,8 @@
 # Generated by Cargo
 # will have compiled files and executables
-/target/
-
+**/target/
+**/*.dat
+**/*.py
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock

diff --git a/.idea/adder-codec-rs.iml b/.idea/adder-codec-rs.iml
diff --git a/.idea/runConfigurations/Run_adder_viz_dynamic.xml b/.idea/runConfigurations/Run_adder_viz_dynamic.xml
diff --git a/adder-codec-core/src/codec/compressed/mod.rs b/adder-codec-core/src/codec/compressed/mod.rs
@@ -65,6 +65,7 @@ mod tests {
             }),
         );
         let meta = *encoder.meta();
+        dbg!(meta);
         encoder.ingest_event(test_event).unwrap();
         test_event.t += 100;
         encoder.ingest_event(test_event).unwrap();

diff --git a/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs b/adder-codec-core/src/codec/compressed/source_model/cabac_contexts.rs
@@ -92,7 +92,11 @@ impl Contexts {
         if t_residual_i64.abs() < self.t_residual_max {
             (0, t_residual_i64)
         } else {
-            let actual_dt = event.t - prev_event.t;
+            if event.t < prev_event.t {
+                // dbg!(event.clone(), prev_event.clone());
+            }
+
+            let actual_dt = event.t.saturating_sub(prev_event.t);
             let actual_intensity = self.event_to_intensity(event.d, actual_dt, dt_ref);
             let mut recon_intensity = actual_intensity;
             let mut bitshift: u8 = 0;

diff --git a/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs b/adder-codec-core/src/codec/compressed/source_model/event_structure/event_cube.rs
@@ -123,8 +123,6 @@ impl HandleEvent for EventCube {
         event.coord.x -= self.start_x;
 
         let item = EventCoordless::from(event);
-        self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()]
-            .push(item);
 
         if self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()]
             .len()
@@ -134,10 +132,17 @@ impl HandleEvent for EventCube {
                 [event.coord.x_usize()][self.raw_event_lists[event.coord.c_usize()]
                 [event.coord.y_usize()][event.coord.x_usize()]
             .len()
-                - 2];
+                - 1];
+            if event.t <= last.t {
+                // dbg!(event.t, last.t);
+                return false;
+            }
             debug_assert!(event.t >= last.t);
         }
 
+        self.raw_event_lists[event.coord.c_usize()][event.coord.y_usize()][event.coord.x_usize()]
+            .push(item);
+
         self.raw_event_memory[event.coord.c_usize()][event.coord.y_usize()]
             [event.coord.x_usize()] = EventCoordless::from(event);
 

diff --git a/adder-codec-core/src/codec/compressed/stream.rs b/adder-codec-core/src/codec/compressed/stream.rs
@@ -1,6 +1,6 @@
 use crate::codec::{CodecError, CodecMetadata, EncoderOptions, ReadCompression, WriteCompression};
 use bitstream_io::{BigEndian, BitRead, BitReader, BitWrite, BitWriter};
-use std::io::{Cursor, Read, Seek, SeekFrom, Write};
+use std::io::{BufWriter, Cursor, Read, Seek, SeekFrom, Write};
 
 use crate::codec::compressed::source_model::event_structure::event_adu::EventAdu;
 use crate::codec::compressed::source_model::HandleEvent;
@@ -27,12 +27,7 @@ pub struct CompressedInput<R: Read> {
 impl<W: Write> CompressedOutput<W> {
     /// Create a new compressed output stream.
     pub fn new(meta: CodecMetadata, writer: W) -> Self {
-        let adu = EventAdu::new(
-            meta.plane,
-            0,
-            meta.ref_interval,
-            (meta.delta_t_max / meta.ref_interval) as usize,
-        );
+        let adu = EventAdu::new(meta.plane, 0, meta.ref_interval, meta.adu_interval as usize);
 
         Self {
             meta,
@@ -132,7 +127,7 @@ impl<W: Write> WriteCompression<W> for CompressedOutput<W> {
 
 impl<R: Read> CompressedInput<R> {
     /// Create a new compressed input stream.
-    pub fn new(delta_t_max: DeltaT, ref_interval: DeltaT) -> Self
+    pub fn new(delta_t_max: DeltaT, ref_interval: DeltaT, adu_interval: usize) -> Self
     where
         Self: Sized,
     {
@@ -147,6 +142,7 @@ impl<R: Read> CompressedInput<R> {
                 delta_t_max,
                 event_size: 0,
                 source_camera: Default::default(),
+                adu_interval,
             },
             adu: None,
             _phantom: std::marker::PhantomData,
@@ -186,7 +182,7 @@ impl<R: Read + Seek> ReadCompression<R> for CompressedInput<R> {
                 self.meta.plane,
                 0,
                 self.meta.ref_interval,
-                (self.meta.delta_t_max / self.meta.ref_interval) as usize,
+                self.meta.adu_interval,
             ));
         }
 
@@ -278,9 +274,10 @@ mod tests {
                 },
                 tps: 7650,
                 ref_interval: dt_ref,
-                delta_t_max: dt_ref * num_intervals as u32,
+                delta_t_max: dt_ref * num_intervals,
                 event_size: 0,
                 source_camera: SourceCamera::FramedU8,
+                adu_interval: num_intervals as usize,
             },
             Cursor::new(Vec::new()),
         );
@@ -338,9 +335,10 @@ mod tests {
                 },
                 tps: 7650,
                 ref_interval: dt_ref,
-                delta_t_max: dt_ref * num_intervals as u32,
+                delta_t_max: dt_ref * num_intervals,
                 event_size: 0,
                 source_camera: SourceCamera::FramedU8,
+                adu_interval: num_intervals as usize,
             },
             Cursor::new(Vec::new()),
         );
@@ -385,7 +383,11 @@ mod tests {
         // Check that the size is less than the raw events
         assert!((output.len() as u32) < counter * 9);
 
-        let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref);
+        let mut compressed_input = CompressedInput::new(
+            dt_ref * num_intervals as u32,
+            dt_ref,
+            num_intervals as usize,
+        );
         compressed_input.meta.plane = plane;
         let mut stream = BitReader::endian(Cursor::new(output), BigEndian);
         for _ in 0..counter - 1 {
@@ -431,6 +433,7 @@ mod tests {
                 delta_t_max: dt_ref * num_intervals as u32,
                 event_size: 0,
                 source_camera: SourceCamera::FramedU8,
+                adu_interval: num_intervals as usize,
             },
             Cursor::new(Vec::new()),
         );
@@ -459,7 +462,11 @@ mod tests {
         // Check that the size is less than the raw events
         assert!((output.len() as u32) < counter * 9);
 
-        let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref);
+        let mut compressed_input = CompressedInput::new(
+            dt_ref * num_intervals as u32,
+            dt_ref,
+            num_intervals as usize,
+        );
         compressed_input.meta.plane = plane;
         let mut stream = BitReader::endian(Cursor::new(output), BigEndian);
         for _ in 0..counter - 1 {
@@ -514,6 +521,7 @@ mod tests {
                 delta_t_max: dt_ref * num_intervals as u32,
                 event_size: 0,
                 source_camera: SourceCamera::FramedU8,
+                adu_interval: num_intervals as usize,
             },
             Cursor::new(Vec::new()),
         );
@@ -571,7 +579,11 @@ mod tests {
         assert!(!output.is_empty());
         // Check that the size is less than the raw events
 
-        let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref);
+        let mut compressed_input = CompressedInput::new(
+            dt_ref * num_intervals as u32,
+            dt_ref,
+            num_intervals as usize,
+        );
         compressed_input.meta.plane = plane;
         let mut stream = BitReader::endian(Cursor::new(output), BigEndian);
         for _ in 0..counter + 1 {
@@ -624,6 +636,7 @@ mod tests {
                 delta_t_max: dt_ref * num_intervals as u32,
                 event_size: 0,
                 source_camera: SourceCamera::FramedU8,
+                adu_interval: num_intervals as usize,
             },
             Cursor::new(Vec::new()),
         );
@@ -688,7 +701,11 @@ mod tests {
         // Check that the size is less than the raw events
         assert!((output.len() as u32) < counter * 9);
 
-        let mut compressed_input = CompressedInput::new(dt_ref * num_intervals as u32, dt_ref);
+        let mut compressed_input = CompressedInput::new(
+            dt_ref * num_intervals as u32,
+            dt_ref,
+            num_intervals as usize,
+        );
         compressed_input.meta.plane = plane;
         let mut stream = BitReader::endian(Cursor::new(output), BigEndian);
         loop {

diff --git a/adder-codec-core/src/codec/decoder.rs b/adder-codec-core/src/codec/decoder.rs
@@ -10,7 +10,8 @@ use crate::codec::compressed::stream::CompressedInput;
 
 use crate::codec::encoder::Encoder;
 use crate::codec::header::{
-    EventStreamHeader, EventStreamHeaderExtensionV1, EventStreamHeaderExtensionV2, MAGIC_COMPRESSED,
+    EventStreamHeader, EventStreamHeaderExtensionV1, EventStreamHeaderExtensionV2,
+    EventStreamHeaderExtensionV3, MAGIC_COMPRESSED,
 };
 use crate::codec::raw::stream::RawInput;
 use crate::codec::CodecError::Deserialize;
@@ -93,7 +94,7 @@ impl<R: Read + Seek> Decoder<R> {
             SourceCamera::FramedU64 => U64,
             SourceCamera::FramedF32 => F32,
             SourceCamera::FramedF64 => F64,
-            SourceCamera::Dvs => F64,
+            SourceCamera::Dvs => U8,
             SourceCamera::DavisU8 => U8,
             SourceCamera::Atis => U8,
             SourceCamera::Asint => F64,
@@ -128,7 +129,8 @@ impl<R: Read + Seek> Decoder<R> {
                 ref_interval: header.ref_interval,
                 delta_t_max: header.delta_t_max,
                 event_size: header.event_size,
-                source_camera: Default::default(),
+                source_camera: Default::default(), // Gets filled by decoding the V2 header extension
+                adu_interval: Default::default(), // Gets filled by decoding the V3 header extension
             };
 
             // Manual fix for malformed files with old software
@@ -183,6 +185,23 @@ impl<R: Read + Seek> Decoder<R> {
             return Ok(());
         }
 
+        extension_size = bincode::serialized_size(&EventStreamHeaderExtensionV3::default())?;
+        buffer = vec![0; extension_size as usize];
+        reader.read_bytes(&mut buffer)?;
+        let extension_v3 = match self
+            .bincode
+            .deserialize_from::<_, EventStreamHeaderExtensionV3>(&*buffer)
+        {
+            Ok(header) => header,
+            Err(_) => return Err(Deserialize),
+        };
+        self.input.meta_mut().adu_interval = extension_v3.adu_interval as usize;
+        self.input.meta_mut().header_size += extension_size as usize;
+
+        if codec_version == 3 {
+            return Ok(());
+        }
+
         Err(CodecError::UnsupportedVersion(codec_version))
     }
 
@@ -289,6 +308,7 @@ mod tests {
                 delta_t_max: 255,
                 event_size: 0,
                 source_camera: Default::default(),
+                adu_interval: 1,
             },
             bufwriter,
         );
@@ -325,6 +345,7 @@ mod tests {
                 delta_t_max: 255,
                 event_size: 0,
                 source_camera: Default::default(),
+                adu_interval: 1,
             },
             bufwriter,
         );
@@ -371,6 +392,7 @@ mod tests {
                 delta_t_max: 255,
                 event_size: 0,
                 source_camera: Default::default(),
+                adu_interval: 1,
             },
             bufwriter,
         );
@@ -436,7 +458,7 @@ mod tests {
         let output = setup_encoded_compressed(0);
         let tmp = Cursor::new(&*output);
         let bufreader = BufReader::new(tmp);
-        let compression = CompressedInput::new(255, 255);
+        let compression = CompressedInput::new(255, 255, 1);
 
         let mut bitreader = BitReader::endian(bufreader, BigEndian);
         let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap();
@@ -449,7 +471,7 @@ mod tests {
         let output = setup_encoded_compressed(1);
         let tmp = Cursor::new(&*output);
         let bufreader = BufReader::new(tmp);
-        let compression = CompressedInput::new(255, 255);
+        let compression = CompressedInput::new(255, 255, 1);
 
         let mut bitreader = BitReader::endian(bufreader, BigEndian);
         let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap();
@@ -462,7 +484,7 @@ mod tests {
         let output = setup_encoded_compressed(2);
         let tmp = Cursor::new(&*output);
         let bufreader = BufReader::new(tmp);
-        let compression = CompressedInput::new(255, 255);
+        let compression = CompressedInput::new(255, 255, 1);
 
         let mut bitreader = BitReader::endian(bufreader, BigEndian);
         let reader = Decoder::new_compressed(compression, &mut bitreader).unwrap();