Skip to content

Commit

Permalink
Dvs fixes (#101)
Browse files Browse the repository at this point in the history 
* Start fixing adder-to-dvs

* Remove OpenCV dependency. Write out videos with video-rs

* update aedat4 viz
  • Loading branch information
@ac-freeman
ac-freeman authored Dec 28, 2023
1 parent 273ef76 commit c7e55bd
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 117 deletions.
4 changes: 2 additions & 2 deletions .idea/runConfigurations/ADDER_to_DVS.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions adder-codec-rs/src/bin_cv/aedat4_dvs_visualize.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use adder_codec_rs::aedat::events_generated::Event;
use adder_codec_rs::transcoder::source::video::show_display_force;
use adder_codec_rs::utils::viz::{encode_video_ffmpeg, write_frame_to_video};
use adder_codec_rs::utils::viz::{encode_video_ffmpeg, write_frame_to_video_cv};
use clap::Parser;
use davis_edi_rs::util::reconstructor::Reconstructor;
use opencv::core::{Mat, MatTrait, MatTraitManual, CV_8U};
Expand Down Expand Up @@ -159,7 +159,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
if args.show_display {
show_display_force("DVS", &frame, 1)?;
}
write_frame_to_video(&frame, &mut video_writer)?;
write_frame_to_video_cv(&frame, &mut video_writer)?;
}
println!("\nDVS event count: {event_count}");
println!("\n");
Expand Down
10 changes: 4 additions & 6 deletions adder-codec-rs/src/utils/viz.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::io::BufWriter;
use std::io::{Cursor, Write};
use std::path::Path;
use std::process::{Command, Output};
use video_rs_adder_dep::Frame;
use video_rs_adder_dep::{Frame, Time};

#[cfg(feature = "open-cv")]
/// Writes a given [`Mat`] to a file
Expand All @@ -21,7 +21,7 @@ use video_rs_adder_dep::Frame;
/// # Panics
/// This function panics if the amount data written to the file is not equal to the amount of data
/// in the [`Mat`].
pub fn write_frame_to_video(
pub fn write_frame_to_video_cv(
frame: &Mat,
video_writer: &mut BufWriter<File>,
) -> Result<(), Box<dyn Error>> {
Expand All @@ -33,10 +33,8 @@ pub fn write_frame_to_video(
unsafe {
for idx in 0..len {
let val: *const u8 = frame.at_unchecked(idx)? as *const u8;
assert_eq!(
video_writer.write(std::slice::from_raw_parts(val, 1))?,
len as usize
);
let bytes_written = video_writer.write(std::slice::from_raw_parts(val, 1))?;
assert_eq!(bytes_written, 1);
}
}
Ok(())
Expand Down
11 changes: 1 addition & 10 deletions adder-to-dvs/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,6 @@ categories = ["multimedia::encoding", "multimedia::video", "science"]

[dependencies]
adder-codec-core = { version = "0.3.0", path = "../adder-codec-core" }
adder-codec-rs = { version = "0.4.0", path = "../adder-codec-rs", features = [
"transcoder",
] }
clap = { version = "4.0.17", features = ["derive"] }
ndarray = { version = "0.15.6", features = ["rayon", "serde"] }

[dependencies.opencv]
version = "0.83.0"
package = 'opencv'
default-features = false
features = ['videoio', 'imgproc', 'highgui', 'clang-runtime']
optional = false
video-rs = { version = "0.5.0", features = ["ndarray"] }
201 changes: 104 additions & 97 deletions adder-to-dvs/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
use adder_codec_core::codec::CodecMetadata;
use adder_codec_core::*;
use adder_codec_rs::transcoder::source::video::show_display_force;
use adder_codec_rs::utils::viz::{encode_video_ffmpeg, write_frame_to_video};
use clap::Parser;
use ndarray::Array3;
use opencv::core::{Mat, MatTrait, MatTraitManual, CV_8U, CV_8UC3};
use std::cmp::max;
use std::collections::VecDeque;
use std::error::Error;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::option::Option;
use std::path::PathBuf;
use std::{error, io};
use video_rs::{Encoder, EncoderSettings, Options, PixelFormat};

/// Command line argument parser
#[derive(Parser, Debug, Default)]
#[derive(Parser, Debug, Default, Clone)]
#[clap(author, version, about, long_about = None)]
pub struct MyArgs {
/// Input ADΔER video path
Expand Down Expand Up @@ -49,11 +49,11 @@ struct DvsPixel {
#[allow(dead_code)]
fn main() -> Result<(), Box<dyn error::Error>> {
let args: MyArgs = MyArgs::parse();
dbg!(args.clone());
let file_path = args.input.as_str();

let output_text_path = args.output_text.as_str();
let output_video_path = args.output_video.as_str();
let raw_path = "./dvs.gray8";

let (mut stream, mut bitreader) = open_file_decoder(file_path)?;

Expand All @@ -72,8 +72,23 @@ fn main() -> Result<(), Box<dyn error::Error>> {

stream.set_input_stream_position(&mut bitreader, first_event_position)?;

let mut video_writer: Option<BufWriter<File>> = match File::create(raw_path) {
Ok(file) => Some(BufWriter::new(file)),
let mut video_writer: Option<Encoder> = match File::create(output_video_path) {
Ok(_) => {
let mut options = std::collections::HashMap::new();
options.insert("crf".to_string(), "0".to_string());
options.insert("preset".to_string(), "veryslow".to_string());
options.insert("qp".to_string(), "0".to_string());
let opts: Options = options.into();

let settings = EncoderSettings::for_h264_custom(
meta.plane.w_usize(),
meta.plane.h_usize(),
PixelFormat::YUV420P,
opts,
);
let encoder = Encoder::new(&PathBuf::from(output_video_path).into(), settings)?;
Some(encoder)
}
Err(_) => None,
};
let mut text_writer: BufWriter<File> = BufWriter::new(File::create(output_text_path)?);
Expand Down Expand Up @@ -115,44 +130,12 @@ fn main() -> Result<(), Box<dyn error::Error>> {
meta.plane.c().into(),
));

let mut instantaneous_frame_deque = {
let mut instantaneous_frame = Mat::default();
match meta.plane.c() {
1 => unsafe {
instantaneous_frame.create_rows_cols(
meta.plane.h() as i32,
meta.plane.w() as i32,
CV_8U,
)?;
},
_ => unsafe {
instantaneous_frame.create_rows_cols(
meta.plane.h() as i32,
meta.plane.w() as i32,
CV_8UC3,
)?;
},
}

VecDeque::from([instantaneous_frame])
};

match instantaneous_frame_deque
.back_mut()
.expect("Could not get back of deque")
.data_bytes_mut()
{
Ok(bytes) => {
for byte in bytes {
*byte = 128;
}
}
Err(e) => {
return Err(Box::new(e));
}
};
let mut instantaneous_frame_deque = VecDeque::from([create_blank_dvs_frame(&meta)?]);

let frame_length = (meta.tps as f32 / args.fps) as u128; // length in ticks
let frame_duration = 1.0 / args.fps as f64; // length in seconds

let mut current_frame_time = 0.0;
let mut frame_count = 0_usize;
let mut current_t = 0;
let mut max_px_event_count = 0;
Expand All @@ -170,13 +153,18 @@ fn main() -> Result<(), Box<dyn error::Error>> {
match instantaneous_frame_deque.pop_front() {
None => {}
Some(frame) => {
if args.show_display {
show_display_force("DVS", &frame, 1)?;
}
// if args.show_display {
// show_display_force("DVS", &frame, 1)?;
// }
match video_writer {
None => {}
Some(ref mut writer) => {
write_frame_to_video(&frame, writer)?;
Some(ref mut encoder) => {
write_frame_to_video(
&frame,
encoder,
video_rs::Time::from_secs_f64(current_frame_time),
)?;
current_frame_time += frame_duration;
}
}
}
Expand All @@ -185,7 +173,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
}

match stream.digest_event(&mut bitreader) {
Ok(event) => {
Ok(mut event) => {
event_count += 1;
let y = event.coord.y as usize;
let x = event.coord.x as usize;
Expand All @@ -208,7 +196,23 @@ fn main() -> Result<(), Box<dyn error::Error>> {
}
},
Some(px) => {
px.t += event.t as u128;
if meta.time_mode == TimeMode::DeltaT {
px.t += event.t as u128;
} else {
let tmp = px.t;
px.t = event.t as u128;
event.t -= tmp as u32;
}

if is_framed(meta.source_camera) {
px.t = if px.t % meta.ref_interval as u128 == 0 {
px.t
} else {
(((px.t / meta.ref_interval as u128) + 1)
* meta.ref_interval as u128)
};
}

current_t = max(px.t, current_t);
let frame_idx = (px.t / frame_length) as usize;

Expand All @@ -228,6 +232,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
// Fire a positive polarity event
set_instant_dvs_pixel(
event,
&meta,
&mut instantaneous_frame_deque,
frame_idx,
frame_count,
Expand All @@ -250,6 +255,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
// Fire a negative polarity event
set_instant_dvs_pixel(
event,
&meta,
&mut instantaneous_frame_deque,
frame_idx,
frame_count,
Expand Down Expand Up @@ -287,34 +293,39 @@ fn main() -> Result<(), Box<dyn error::Error>> {

let mut event_count_mat = instantaneous_frame_deque[0].clone();
unsafe {
for y in 0..meta.plane.h() as i32 {
for x in 0..meta.plane.w() as i32 {
for c in 0..meta.plane.c() as i32 {
*event_count_mat.at_3d_unchecked_mut(y, x, c)? =
((event_counts[[y as usize, x as usize, c as usize]] as f32
/ max_px_event_count as f32)
* 255.0) as u8;
for y in 0..meta.plane.h_usize() {
for x in 0..meta.plane.w_usize() {
for c in 0..meta.plane.c_usize() {
event_count_mat[[y, x, c]] = ((event_counts[[y, x, c]] as f32
/ max_px_event_count as f32)
* 255.0) as u8;
}
}
}
}

for frame in instantaneous_frame_deque {
if args.show_display {
show_display_force("DVS", &frame, 1)?;
}
// if args.show_display {
// show_display_force("DVS", &frame, 1)?;
// }
match video_writer {
None => {}
Some(ref mut writer) => {
write_frame_to_video(&frame, writer)?;
Some(ref mut encoder) => {
write_frame_to_video(
&frame,
encoder,
video_rs::Time::from_secs_f64(current_frame_time),
)?;
current_frame_time += frame_duration;
}
}
}
println!("\n");
if args.show_display {
show_display_force("Event counts", &event_count_mat, 0)?;
}
encode_video_ffmpeg(raw_path, output_video_path)?;

// TODO: restore this functionality
// if args.show_display {
// show_display_force("Event counts", &event_count_mat, 0)?;
// }

handle.flush()?;
println!("Finished!");
Expand All @@ -323,7 +334,8 @@ fn main() -> Result<(), Box<dyn error::Error>> {

fn set_instant_dvs_pixel(
event: Event,
frames: &mut VecDeque<Mat>,
meta: &CodecMetadata,
frames: &mut VecDeque<Array3<u8>>,
frame_idx: usize,
frame_count: usize,
value: u128,
Expand All @@ -332,39 +344,18 @@ fn set_instant_dvs_pixel(
let grow_len = frame_idx as i32 - frame_count as i32 - frames.len() as i32 + 1;

for _ in 0..grow_len {
frames.push_back(frames[0].clone());
// Clear the instantaneous frame
match frames
.back_mut()
.expect("Could not get back of deque")
.data_bytes_mut()
{
Ok(bytes) => {
for byte in bytes {
*byte = 128;
}
}
Err(e) => {
return Err(e.into());
}
};
frames.push_back(create_blank_dvs_frame(&meta)?);
}

unsafe {
let px: &mut u8 = match event.coord.c {
None => frames[frame_idx - frame_count]
.at_2d_unchecked_mut(event.coord.y.into(), event.coord.x.into())?,
Some(c) => frames[frame_idx - frame_count].at_3d_unchecked_mut(
event.coord.y.into(),
event.coord.x.into(),
c.into(),
)?,
};
*px = value as u8;
// match value {
// 128 => *px = 128,
// a => *px = (*px as i16 + a) as u8,
// }
if frame_idx >= frame_count {
frames[frame_idx - frame_count][[event.coord.y.into(), event.coord.x.into(), 0]] =
value as u8;
frames[frame_idx - frame_count][[event.coord.y.into(), event.coord.x.into(), 1]] =
value as u8;
frames[frame_idx - frame_count][[event.coord.y.into(), event.coord.x.into(), 2]] =
value as u8;
}
}
Ok(())
}
Expand All @@ -379,3 +370,19 @@ fn event_to_frame_intensity(event: &Event, frame_length: u128) -> f64 {
.ln_1p(),
}
}

fn create_blank_dvs_frame(meta: &CodecMetadata) -> Result<Array3<u8>, Box<dyn Error>> {
let instantaneous_frame: Array3<u8> = Array3::from_shape_vec(
(meta.plane.h_usize(), meta.plane.w_usize(), 3),
vec![128_u8; meta.plane.h_usize() * meta.plane.w_usize() * 3],
)?;
Ok(instantaneous_frame)
}

pub fn write_frame_to_video(
frame: &video_rs::Frame,
encoder: &mut video_rs::Encoder,
timestamp: video_rs::Time,
) -> Result<(), Box<dyn Error>> {
encoder.encode(&frame, &timestamp).map_err(|e| e.into())
}

0 comments on commit c7e55bd

Please sign in to comment.